2024-12-10 01:14:16,801 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 01:14:16,816 main DEBUG Took 0.012114 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 01:14:16,816 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 01:14:16,817 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 01:14:16,818 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 01:14:16,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,828 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 01:14:16,843 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,845 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,846 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,847 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,848 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,848 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,850 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,851 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,852 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,853 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,854 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,855 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,856 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 01:14:16,857 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,857 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 01:14:16,859 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 01:14:16,861 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 01:14:16,863 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 01:14:16,864 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 01:14:16,865 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 01:14:16,866 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 01:14:16,876 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 01:14:16,880 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 01:14:16,882 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 01:14:16,882 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 01:14:16,883 main DEBUG createAppenders(={Console}) 2024-12-10 01:14:16,884 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-10 01:14:16,884 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 01:14:16,884 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-10 01:14:16,885 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 01:14:16,885 main DEBUG OutputStream closed 2024-12-10 01:14:16,886 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 01:14:16,886 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 01:14:16,886 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-10 01:14:16,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 01:14:16,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 01:14:16,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 01:14:16,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 01:14:16,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 01:14:16,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 01:14:16,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 01:14:16,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 01:14:16,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 01:14:16,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 01:14:16,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 01:14:16,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 01:14:16,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 01:14:16,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 01:14:16,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 01:14:16,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 01:14:16,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 01:14:16,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 01:14:16,963 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 01:14:16,963 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-10 01:14:16,964 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 01:14:16,964 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-10T01:14:17,175 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d 2024-12-10 01:14:17,177 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 01:14:17,178 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T01:14:17,186 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-10T01:14:17,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T01:14:17,221 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54, deleteOnExit=true 2024-12-10T01:14:17,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T01:14:17,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/test.cache.data in system properties and HBase conf 2024-12-10T01:14:17,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T01:14:17,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.log.dir in system properties and HBase conf 2024-12-10T01:14:17,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T01:14:17,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T01:14:17,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T01:14:17,301 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T01:14:17,394 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T01:14:17,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T01:14:17,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T01:14:17,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T01:14:17,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T01:14:17,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T01:14:17,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T01:14:17,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T01:14:17,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T01:14:17,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T01:14:17,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/nfs.dump.dir in system properties and HBase conf 2024-12-10T01:14:17,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/java.io.tmpdir in system properties and HBase conf 2024-12-10T01:14:17,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T01:14:17,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T01:14:17,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T01:14:18,588 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T01:14:18,668 INFO [Time-limited test {}] log.Log(170): Logging initialized @2608ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T01:14:18,737 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T01:14:18,814 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T01:14:18,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T01:14:18,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T01:14:18,842 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T01:14:18,860 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T01:14:18,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b490999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.log.dir/,AVAILABLE} 2024-12-10T01:14:18,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@707ebc91{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T01:14:19,055 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f062aca{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/java.io.tmpdir/jetty-localhost-45341-hadoop-hdfs-3_4_1-tests_jar-_-any-6178512522721247107/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T01:14:19,063 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4fee885d{HTTP/1.1, (http/1.1)}{localhost:45341} 2024-12-10T01:14:19,063 INFO [Time-limited test {}] server.Server(415): Started @3004ms 2024-12-10T01:14:19,698 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T01:14:19,706 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T01:14:19,707 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T01:14:19,707 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T01:14:19,707 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T01:14:19,708 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@849d63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.log.dir/,AVAILABLE} 2024-12-10T01:14:19,709 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a247cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T01:14:19,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@53343f2d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/java.io.tmpdir/jetty-localhost-39737-hadoop-hdfs-3_4_1-tests_jar-_-any-17850932782152395567/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T01:14:19,821 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ff8ab72{HTTP/1.1, (http/1.1)}{localhost:39737} 2024-12-10T01:14:19,821 INFO [Time-limited test {}] server.Server(415): Started @3762ms 2024-12-10T01:14:19,873 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T01:14:20,964 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/data/data2/current/BP-1924572727-172.17.0.3-1733793257899/current, will proceed with Du for space computation calculation, 2024-12-10T01:14:20,964 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/data/data1/current/BP-1924572727-172.17.0.3-1733793257899/current, will proceed with Du for space computation calculation, 2024-12-10T01:14:20,988 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T01:14:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x528b075e62b1596f with lease ID 0xc306c2aef1c5d90a: Processing first storage report for DS-f4d9d2cb-ddd6-461a-b0b2-bd662bd44240 from datanode DatanodeRegistration(127.0.0.1:32797, datanodeUuid=e46bf357-c9e1-452a-b245-f91dfd1874c4, infoPort=39073, infoSecurePort=0, ipcPort=36863, storageInfo=lv=-57;cid=testClusterID;nsid=93230425;c=1733793257899) 2024-12-10T01:14:21,036 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x528b075e62b1596f with lease ID 0xc306c2aef1c5d90a: from storage DS-f4d9d2cb-ddd6-461a-b0b2-bd662bd44240 node DatanodeRegistration(127.0.0.1:32797, datanodeUuid=e46bf357-c9e1-452a-b245-f91dfd1874c4, infoPort=39073, infoSecurePort=0, ipcPort=36863, storageInfo=lv=-57;cid=testClusterID;nsid=93230425;c=1733793257899), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T01:14:21,037 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x528b075e62b1596f with lease ID 0xc306c2aef1c5d90a: Processing first storage report for DS-7830f51e-2083-4ad3-9805-18e86035fefe from datanode DatanodeRegistration(127.0.0.1:32797, datanodeUuid=e46bf357-c9e1-452a-b245-f91dfd1874c4, infoPort=39073, infoSecurePort=0, ipcPort=36863, storageInfo=lv=-57;cid=testClusterID;nsid=93230425;c=1733793257899) 2024-12-10T01:14:21,037 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x528b075e62b1596f with lease ID 0xc306c2aef1c5d90a: from storage DS-7830f51e-2083-4ad3-9805-18e86035fefe node DatanodeRegistration(127.0.0.1:32797, datanodeUuid=e46bf357-c9e1-452a-b245-f91dfd1874c4, infoPort=39073, infoSecurePort=0, ipcPort=36863, storageInfo=lv=-57;cid=testClusterID;nsid=93230425;c=1733793257899), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T01:14:21,054 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d 2024-12-10T01:14:21,115 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/zookeeper_0, clientPort=49530, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T01:14:21,123 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49530 2024-12-10T01:14:21,132 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:21,135 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:21,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741825_1001 (size=7) 2024-12-10T01:14:21,744 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 with version=8 2024-12-10T01:14:21,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/hbase-staging 2024-12-10T01:14:21,820 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T01:14:22,038 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fd1a2b66caf8:0 server-side Connection retries=45 2024-12-10T01:14:22,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,051 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T01:14:22,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T01:14:22,168 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T01:14:22,220 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T01:14:22,228 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T01:14:22,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T01:14:22,255 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4817 (auto-detected) 2024-12-10T01:14:22,256 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-10T01:14:22,276 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37249 2024-12-10T01:14:22,298 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37249 connecting to ZooKeeper ensemble=127.0.0.1:49530 2024-12-10T01:14:22,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:372490x0, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T01:14:22,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37249-0x1000d49acf30000 connected 2024-12-10T01:14:22,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:22,509 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:22,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T01:14:22,523 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126, hbase.cluster.distributed=false 2024-12-10T01:14:22,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T01:14:22,550 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37249 2024-12-10T01:14:22,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37249 2024-12-10T01:14:22,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37249 2024-12-10T01:14:22,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37249 2024-12-10T01:14:22,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37249 2024-12-10T01:14:22,641 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fd1a2b66caf8:0 server-side Connection retries=45 2024-12-10T01:14:22,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,643 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T01:14:22,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T01:14:22,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T01:14:22,646 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T01:14:22,648 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T01:14:22,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38925 2024-12-10T01:14:22,651 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38925 connecting to ZooKeeper ensemble=127.0.0.1:49530 2024-12-10T01:14:22,652 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:22,654 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:22,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389250x0, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T01:14:22,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:389250x0, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T01:14:22,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38925-0x1000d49acf30001 connected 2024-12-10T01:14:22,675 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T01:14:22,682 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T01:14:22,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T01:14:22,688 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T01:14:22,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38925 2024-12-10T01:14:22,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38925 2024-12-10T01:14:22,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38925 2024-12-10T01:14:22,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38925 2024-12-10T01:14:22,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38925 2024-12-10T01:14:22,705 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fd1a2b66caf8:37249 2024-12-10T01:14:22,706 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:22,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T01:14:22,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T01:14:22,725 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:22,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:22,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T01:14:22,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:22,756 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T01:14:22,757 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fd1a2b66caf8,37249,1733793261894 from backup master directory 2024-12-10T01:14:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T01:14:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:22,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T01:14:22,766 WARN [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T01:14:22,767 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:22,769 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T01:14:22,770 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T01:14:22,820 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/hbase.id] with ID: 97129feb-7bb1-48e1-a470-4d1c86822c6e 2024-12-10T01:14:22,821 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/.tmp/hbase.id 2024-12-10T01:14:22,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741826_1002 (size=42) 2024-12-10T01:14:23,235 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/.tmp/hbase.id]:[hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/hbase.id] 2024-12-10T01:14:23,285 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T01:14:23,289 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T01:14:23,305 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-10T01:14:23,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:23,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:23,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741827_1003 (size=196) 2024-12-10T01:14:23,795 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:14:23,797 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T01:14:23,810 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:23,813 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T01:14:23,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741828_1004 (size=1189) 2024-12-10T01:14:24,266 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store 2024-12-10T01:14:24,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741829_1005 (size=34) 2024-12-10T01:14:24,697 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T01:14:24,700 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:24,701 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T01:14:24,702 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:14:24,702 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:14:24,704 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T01:14:24,704 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:14:24,704 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:14:24,705 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733793264701Disabling compacts and flushes for region at 1733793264701Disabling writes for close at 1733793264704 (+3 ms)Writing region close event to WAL at 1733793264704Closed at 1733793264704 2024-12-10T01:14:24,707 WARN [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/.initializing 2024-12-10T01:14:24,707 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/WALs/fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:24,715 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T01:14:24,728 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd1a2b66caf8%2C37249%2C1733793261894, suffix=, logDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/WALs/fd1a2b66caf8,37249,1733793261894, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/oldWALs, maxLogs=10 2024-12-10T01:14:24,748 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/WALs/fd1a2b66caf8,37249,1733793261894/fd1a2b66caf8%2C37249%2C1733793261894.1733793264732, exclude list is [], retry=0 2024-12-10T01:14:24,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32797,DS-f4d9d2cb-ddd6-461a-b0b2-bd662bd44240,DISK] 2024-12-10T01:14:24,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T01:14:24,796 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/WALs/fd1a2b66caf8,37249,1733793261894/fd1a2b66caf8%2C37249%2C1733793261894.1733793264732 2024-12-10T01:14:24,797 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39073:39073)] 2024-12-10T01:14:24,798 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:14:24,798 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:24,801 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,802 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T01:14:24,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:24,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:24,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T01:14:24,865 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:24,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:24,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T01:14:24,869 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:24,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:24,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T01:14:24,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:24,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:24,874 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,877 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,878 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,883 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,883 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,887 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T01:14:24,890 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T01:14:24,894 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:14:24,895 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63668130, jitterRate=-0.05127093195915222}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T01:14:24,900 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733793264812Initializing all the Stores at 1733793264814 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793264815 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793264815Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793264816 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793264816Cleaning up temporary data from old regions at 1733793264883 (+67 ms)Region opened successfully at 1733793264900 (+17 ms) 2024-12-10T01:14:24,901 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T01:14:24,929 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3466340d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fd1a2b66caf8/172.17.0.3:0 2024-12-10T01:14:24,956 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T01:14:24,965 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T01:14:24,965 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T01:14:24,967 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T01:14:24,968 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T01:14:24,973 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-10T01:14:24,973 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T01:14:24,995 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T01:14:25,003 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T01:14:25,082 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T01:14:25,087 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T01:14:25,090 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T01:14:25,102 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T01:14:25,105 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T01:14:25,110 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T01:14:25,121 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T01:14:25,123 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T01:14:25,134 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T01:14:25,152 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T01:14:25,163 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T01:14:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T01:14:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T01:14:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,180 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fd1a2b66caf8,37249,1733793261894, sessionid=0x1000d49acf30000, setting cluster-up flag (Was=false) 2024-12-10T01:14:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,240 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T01:14:25,243 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:25,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:25,302 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T01:14:25,306 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:25,314 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T01:14:25,380 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T01:14:25,389 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T01:14:25,395 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T01:14:25,397 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(746): ClusterId : 97129feb-7bb1-48e1-a470-4d1c86822c6e 2024-12-10T01:14:25,399 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T01:14:25,401 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fd1a2b66caf8,37249,1733793261894 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T01:14:25,409 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T01:14:25,409 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T01:14:25,410 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fd1a2b66caf8:0, corePoolSize=5, maxPoolSize=5 2024-12-10T01:14:25,410 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fd1a2b66caf8:0, corePoolSize=5, maxPoolSize=5 2024-12-10T01:14:25,411 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fd1a2b66caf8:0, corePoolSize=5, maxPoolSize=5 2024-12-10T01:14:25,411 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fd1a2b66caf8:0, corePoolSize=5, maxPoolSize=5 2024-12-10T01:14:25,411 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fd1a2b66caf8:0, corePoolSize=10, maxPoolSize=10 2024-12-10T01:14:25,411 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,411 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fd1a2b66caf8:0, corePoolSize=2, maxPoolSize=2 2024-12-10T01:14:25,412 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,413 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733793295413 2024-12-10T01:14:25,414 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T01:14:25,415 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T01:14:25,416 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T01:14:25,417 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T01:14:25,419 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T01:14:25,419 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T01:14:25,420 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T01:14:25,420 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T01:14:25,420 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T01:14:25,420 DEBUG [RS:0;fd1a2b66caf8:38925 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f2eb57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fd1a2b66caf8/172.17.0.3:0 2024-12-10T01:14:25,421 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,424 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:25,425 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T01:14:25,424 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T01:14:25,426 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T01:14:25,426 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T01:14:25,429 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T01:14:25,429 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T01:14:25,433 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.large.0-1733793265431,5,FailOnTimeoutGroup] 2024-12-10T01:14:25,433 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.small.0-1733793265433,5,FailOnTimeoutGroup] 2024-12-10T01:14:25,433 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,434 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T01:14:25,435 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741831_1007 (size=1321) 2024-12-10T01:14:25,436 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,436 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fd1a2b66caf8:38925 2024-12-10T01:14:25,439 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T01:14:25,439 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T01:14:25,439 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T01:14:25,442 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(2659): reportForDuty to master=fd1a2b66caf8,37249,1733793261894 with port=38925, startcode=1733793262612 2024-12-10T01:14:25,452 DEBUG [RS:0;fd1a2b66caf8:38925 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T01:14:25,481 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T01:14:25,487 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37249 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,489 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37249 {}] master.ServerManager(517): Registering regionserver=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,501 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:14:25,501 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35831 2024-12-10T01:14:25,501 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T01:14:25,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T01:14:25,513 DEBUG [RS:0;fd1a2b66caf8:38925 {}] zookeeper.ZKUtil(111): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,513 WARN [RS:0;fd1a2b66caf8:38925 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T01:14:25,514 INFO [RS:0;fd1a2b66caf8:38925 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T01:14:25,514 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,516 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fd1a2b66caf8,38925,1733793262612] 2024-12-10T01:14:25,539 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T01:14:25,550 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T01:14:25,554 INFO [RS:0;fd1a2b66caf8:38925 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T01:14:25,554 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,555 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T01:14:25,560 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T01:14:25,561 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,561 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,561 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,561 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,561 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fd1a2b66caf8:0, corePoolSize=2, maxPoolSize=2 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,562 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,563 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fd1a2b66caf8:0, corePoolSize=1, maxPoolSize=1 2024-12-10T01:14:25,563 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fd1a2b66caf8:0, corePoolSize=3, maxPoolSize=3 2024-12-10T01:14:25,563 DEBUG [RS:0;fd1a2b66caf8:38925 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0, corePoolSize=3, maxPoolSize=3 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,564 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,38925,1733793262612-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T01:14:25,580 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T01:14:25,581 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,38925,1733793262612-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,582 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,582 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.Replication(171): fd1a2b66caf8,38925,1733793262612 started 2024-12-10T01:14:25,596 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:25,597 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1482): Serving as fd1a2b66caf8,38925,1733793262612, RpcServer on fd1a2b66caf8/172.17.0.3:38925, sessionid=0x1000d49acf30001 2024-12-10T01:14:25,597 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T01:14:25,597 DEBUG [RS:0;fd1a2b66caf8:38925 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,598 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fd1a2b66caf8,38925,1733793262612' 2024-12-10T01:14:25,598 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fd1a2b66caf8,38925,1733793262612' 2024-12-10T01:14:25,599 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T01:14:25,600 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T01:14:25,601 DEBUG [RS:0;fd1a2b66caf8:38925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T01:14:25,601 INFO [RS:0;fd1a2b66caf8:38925 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T01:14:25,601 INFO [RS:0;fd1a2b66caf8:38925 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T01:14:25,710 INFO [RS:0;fd1a2b66caf8:38925 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T01:14:25,713 INFO [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd1a2b66caf8%2C38925%2C1733793262612, suffix=, logDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/oldWALs, maxLogs=32 2024-12-10T01:14:25,732 DEBUG [RS:0;fd1a2b66caf8:38925 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612/fd1a2b66caf8%2C38925%2C1733793262612.1733793265715, exclude list is [], retry=0 2024-12-10T01:14:25,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32797,DS-f4d9d2cb-ddd6-461a-b0b2-bd662bd44240,DISK] 2024-12-10T01:14:25,739 INFO [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612/fd1a2b66caf8%2C38925%2C1733793262612.1733793265715 2024-12-10T01:14:25,739 DEBUG [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39073:39073)] 2024-12-10T01:14:25,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T01:14:25,841 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:14:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741833_1009 (size=32) 2024-12-10T01:14:26,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:26,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T01:14:26,268 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T01:14:26,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T01:14:26,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T01:14:26,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T01:14:26,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T01:14:26,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T01:14:26,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T01:14:26,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T01:14:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740 2024-12-10T01:14:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740 2024-12-10T01:14:26,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T01:14:26,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T01:14:26,289 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:14:26,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T01:14:26,294 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:14:26,295 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61269572, jitterRate=-0.08701223134994507}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:14:26,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733793266261Initializing all the Stores at 1733793266265 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266265Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266265Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793266265Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266265Cleaning up temporary data from old regions at 1733793266288 (+23 ms)Region opened successfully at 1733793266297 (+9 ms) 2024-12-10T01:14:26,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T01:14:26,297 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T01:14:26,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T01:14:26,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T01:14:26,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T01:14:26,299 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T01:14:26,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733793266297Disabling compacts and flushes for region at 1733793266297Disabling writes for close at 1733793266297Writing region close event to WAL at 1733793266298 (+1 ms)Closed at 1733793266299 (+1 ms) 2024-12-10T01:14:26,302 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T01:14:26,302 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T01:14:26,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T01:14:26,314 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T01:14:26,316 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T01:14:26,474 DEBUG [fd1a2b66caf8:37249 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T01:14:26,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:26,491 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fd1a2b66caf8,38925,1733793262612, state=OPENING 2024-12-10T01:14:26,555 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T01:14:26,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:26,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:14:26,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T01:14:26,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T01:14:26,647 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T01:14:26,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:26,826 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T01:14:26,829 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51129, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T01:14:26,839 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T01:14:26,840 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T01:14:26,840 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T01:14:26,843 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd1a2b66caf8%2C38925%2C1733793262612.meta, suffix=.meta, logDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/oldWALs, maxLogs=32 2024-12-10T01:14:26,856 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612/fd1a2b66caf8%2C38925%2C1733793262612.meta.1733793266845.meta, exclude list is [], retry=0 2024-12-10T01:14:26,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32797,DS-f4d9d2cb-ddd6-461a-b0b2-bd662bd44240,DISK] 2024-12-10T01:14:26,862 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/WALs/fd1a2b66caf8,38925,1733793262612/fd1a2b66caf8%2C38925%2C1733793262612.meta.1733793266845.meta 2024-12-10T01:14:26,863 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39073:39073)] 2024-12-10T01:14:26,863 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:14:26,865 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T01:14:26,867 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T01:14:26,871 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T01:14:26,875 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T01:14:26,876 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:26,876 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T01:14:26,876 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T01:14:26,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T01:14:26,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T01:14:26,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T01:14:26,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T01:14:26,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T01:14:26,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T01:14:26,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T01:14:26,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T01:14:26,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:26,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T01:14:26,889 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T01:14:26,891 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740 2024-12-10T01:14:26,894 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740 2024-12-10T01:14:26,896 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T01:14:26,896 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T01:14:26,897 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:14:26,900 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T01:14:26,902 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74862956, jitterRate=0.11554497480392456}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:14:26,902 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T01:14:26,903 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733793266877Writing region info on filesystem at 1733793266877Initializing all the Stores at 1733793266878 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266879 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266879Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793266879Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733793266879Cleaning up temporary data from old regions at 1733793266896 (+17 ms)Running coprocessor post-open hooks at 1733793266902 (+6 ms)Region opened successfully at 1733793266903 (+1 ms) 2024-12-10T01:14:26,909 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733793266816 2024-12-10T01:14:26,918 DEBUG [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T01:14:26,919 INFO [RS_OPEN_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T01:14:26,920 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:26,922 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fd1a2b66caf8,38925,1733793262612, state=OPEN 2024-12-10T01:14:26,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T01:14:26,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T01:14:26,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T01:14:26,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T01:14:26,978 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:26,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T01:14:26,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fd1a2b66caf8,38925,1733793262612 in 329 msec 2024-12-10T01:14:26,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T01:14:26,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 683 msec 2024-12-10T01:14:26,997 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T01:14:26,998 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T01:14:27,018 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:27,019 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:27,036 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:27,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:27,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7150 sec 2024-12-10T01:14:27,058 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733793267058, completionTime=-1 2024-12-10T01:14:27,061 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T01:14:27,061 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T01:14:27,084 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-10T01:14:27,084 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733793327084 2024-12-10T01:14:27,084 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733793387084 2024-12-10T01:14:27,084 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-10T01:14:27,086 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,086 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,086 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,088 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fd1a2b66caf8:37249, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,088 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,088 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,095 DEBUG [master/fd1a2b66caf8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T01:14:27,116 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.348sec 2024-12-10T01:14:27,117 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T01:14:27,118 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T01:14:27,119 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T01:14:27,119 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T01:14:27,119 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T01:14:27,120 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T01:14:27,121 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T01:14:27,127 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T01:14:27,128 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T01:14:27,129 INFO [master/fd1a2b66caf8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd1a2b66caf8,37249,1733793261894-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T01:14:27,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d3128f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:27,213 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T01:14:27,213 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T01:14:27,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:27,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:27,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b43726f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:27,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:27,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c08b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:27,236 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:27,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:27,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:27,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=420, ProcessCount=11, AvailableMemoryMB=7953 2024-12-10T01:14:27,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6fcc49df 2024-12-10T01:14:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:14:27,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:14:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:14:27,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:14:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:27,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:14:27,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-10T01:14:27,325 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:27,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:14:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:27,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741835_1011 (size=1061) 2024-12-10T01:14:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:27,749 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:14:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741836_1012 (size=53) 2024-12-10T01:14:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:28,161 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:28,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5c85b06919b802618bd08e23219e27af, disabling compactions & flushes 2024-12-10T01:14:28,162 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,162 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,163 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. after waiting 0 ms 2024-12-10T01:14:28,163 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,163 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c85b06919b802618bd08e23219e27af: Waiting for close lock at 1733793268162Disabling compacts and flushes for region at 1733793268162Disabling writes for close at 1733793268163 (+1 ms)Writing region close event to WAL at 1733793268163Closed at 1733793268163 2024-12-10T01:14:28,169 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:14:28,177 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793268169"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793268169"}]},"ts":"1733793268169"} 2024-12-10T01:14:28,181 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:14:28,183 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:14:28,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793268183"}]},"ts":"1733793268183"} 2024-12-10T01:14:28,190 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:14:28,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, ASSIGN}] 2024-12-10T01:14:28,194 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, ASSIGN 2024-12-10T01:14:28,196 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:14:28,350 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5c85b06919b802618bd08e23219e27af, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:28,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, ASSIGN because future has completed 2024-12-10T01:14:28,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:28,524 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,525 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:14:28,525 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,526 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:28,526 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,526 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,529 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,533 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:28,533 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c85b06919b802618bd08e23219e27af columnFamilyName A 2024-12-10T01:14:28,533 DEBUG [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:28,534 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(327): Store=5c85b06919b802618bd08e23219e27af/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:28,535 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,537 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:28,537 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c85b06919b802618bd08e23219e27af columnFamilyName B 2024-12-10T01:14:28,538 DEBUG [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:28,538 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(327): Store=5c85b06919b802618bd08e23219e27af/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:28,539 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,541 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:28,541 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c85b06919b802618bd08e23219e27af columnFamilyName C 2024-12-10T01:14:28,541 DEBUG [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:28,542 INFO [StoreOpener-5c85b06919b802618bd08e23219e27af-1 {}] regionserver.HStore(327): Store=5c85b06919b802618bd08e23219e27af/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:28,543 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,543 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,544 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,545 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,547 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,547 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,548 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:14:28,550 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,554 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:14:28,555 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5c85b06919b802618bd08e23219e27af; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66384164, jitterRate=-0.010798871517181396}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:14:28,555 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:28,556 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5c85b06919b802618bd08e23219e27af: Running coprocessor pre-open hook at 1733793268526Writing region info on filesystem at 1733793268526Initializing all the Stores at 1733793268528 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793268528Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793268529 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793268529Cleaning up temporary data from old regions at 1733793268547 (+18 ms)Running coprocessor post-open hooks at 1733793268555 (+8 ms)Region opened successfully at 1733793268556 (+1 ms) 2024-12-10T01:14:28,558 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., pid=6, masterSystemTime=1733793268514 2024-12-10T01:14:28,562 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,562 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:28,564 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5c85b06919b802618bd08e23219e27af, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:28,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:14:28,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T01:14:28,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 in 210 msec 2024-12-10T01:14:28,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T01:14:28,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, ASSIGN in 381 msec 2024-12-10T01:14:28,579 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:14:28,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793268580"}]},"ts":"1733793268580"} 2024-12-10T01:14:28,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:14:28,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:14:28,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-12-10T01:14:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T01:14:29,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:29,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44898a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7880fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515395e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:14:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5522ebb1 2024-12-10T01:14:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:14:29,523 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:14:29,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265b3055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,529 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4545009e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,533 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bd1ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776dcc7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ede97e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,540 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d903666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b83180b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@818d592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac348af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6167b64c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,550 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63473470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,554 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d8a1d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504dd62a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,557 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7512cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,559 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300379dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8c4061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102d4bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f6ca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16aad90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34feac5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65dbaac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3461ebf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,575 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8072d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cba810e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29322cfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:29,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:29,581 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:29,583 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354728a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:29,589 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,589 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,589 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,589 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,590 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,590 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,590 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,591 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,593 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,594 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,594 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:29,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:29,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:29,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:29,599 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:29,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-10T01:14:29,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,614 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:14:29,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:29,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:14:29,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:29,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:29,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4f377a73693442fd96009980f2d8893e is 50, key is test_row_0/A:col10/1733793269663/Put/seqid=0 2024-12-10T01:14:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:29,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:29,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:29,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793329778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793329771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793329799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793269756, max timestamp=1733793269797Num uniques 859; 2024-12-10T01:14:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:29,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741837_1013 (size=16681) 2024-12-10T01:14:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793269756, max timestamp=1733793269797Num uniques 859; 2024-12-10T01:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:29,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4f377a73693442fd96009980f2d8893e 2024-12-10T01:14:29,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793269756, max timestamp=1733793269797Num uniques 859; 2024-12-10T01:14:29,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793329955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/7838763eac134fc8859d2af6bfb7acfc is 50, key is test_row_0/B:col10/1733793269663/Put/seqid=0 2024-12-10T01:14:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793329960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793329963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:29,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793329970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793329971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741838_1014 (size=12001) 2024-12-10T01:14:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:30,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/7838763eac134fc8859d2af6bfb7acfc 2024-12-10T01:14:30,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddae1a27f56c409b93b514c7989685be is 50, key is test_row_0/C:col10/1733793269663/Put/seqid=0 2024-12-10T01:14:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793330081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793330081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741839_1015 (size=12001) 2024-12-10T01:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793330081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793330094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddae1a27f56c409b93b514c7989685be 2024-12-10T01:14:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793330095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4f377a73693442fd96009980f2d8893e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e 2024-12-10T01:14:30,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e, entries=250, sequenceid=13, filesize=16.3 K 2024-12-10T01:14:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/7838763eac134fc8859d2af6bfb7acfc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc 2024-12-10T01:14:30,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T01:14:30,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddae1a27f56c409b93b514c7989685be as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be 2024-12-10T01:14:30,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T01:14:30,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 5c85b06919b802618bd08e23219e27af in 527ms, sequenceid=13, compaction requested=false 2024-12-10T01:14:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T01:14:30,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:30,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T01:14:30,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:30,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:30,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:30,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:30,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:30,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:30,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:30,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/feecb7a687714e3789ed3a2cf80b4de4 is 50, key is test_row_0/A:col10/1733793270312/Put/seqid=0 2024-12-10T01:14:30,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741840_1016 (size=14341) 2024-12-10T01:14:30,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/feecb7a687714e3789ed3a2cf80b4de4 2024-12-10T01:14:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793330345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793330352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793330353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793330359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793330346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c22f611cc0ca4ef992829099d29336d5 is 50, key is test_row_0/B:col10/1733793270312/Put/seqid=0 2024-12-10T01:14:30,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741841_1017 (size=12001) 2024-12-10T01:14:30,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c22f611cc0ca4ef992829099d29336d5 2024-12-10T01:14:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793330481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f1f7a8b23ffe4f91989473daeeab1090 is 50, key is test_row_0/C:col10/1733793270312/Put/seqid=0 2024-12-10T01:14:30,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:30,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793330503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741842_1018 (size=12001) 2024-12-10T01:14:30,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f1f7a8b23ffe4f91989473daeeab1090 2024-12-10T01:14:30,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/feecb7a687714e3789ed3a2cf80b4de4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4 2024-12-10T01:14:30,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4, entries=200, sequenceid=50, filesize=14.0 K 2024-12-10T01:14:30,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c22f611cc0ca4ef992829099d29336d5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5 2024-12-10T01:14:30,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T01:14:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f1f7a8b23ffe4f91989473daeeab1090 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090 2024-12-10T01:14:30,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T01:14:30,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5c85b06919b802618bd08e23219e27af in 290ms, sequenceid=50, compaction requested=false 2024-12-10T01:14:30,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:30,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T01:14:30,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:30,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4dc14f59834440afae0a7e38ac344286 is 50, key is test_row_0/A:col10/1733793270340/Put/seqid=0 2024-12-10T01:14:30,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741843_1019 (size=9657) 2024-12-10T01:14:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:30,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:30,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793270736, max timestamp=1733793270763Num uniques 859; 2024-12-10T01:14:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793330765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793270736, max timestamp=1733793270763Num uniques 859; 2024-12-10T01:14:30,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:30,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793270736, max timestamp=1733793270777Num uniques 859; 2024-12-10T01:14:30,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793330850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793330853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793330854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793330856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793330890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793330970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793330981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793330981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793330981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4dc14f59834440afae0a7e38ac344286 2024-12-10T01:14:31,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793331101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/aa2c3e0f4f72419e9ac71fb3c9817c36 is 50, key is test_row_0/B:col10/1733793270340/Put/seqid=0 2024-12-10T01:14:31,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741844_1020 (size=9657) 2024-12-10T01:14:31,171 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/aa2c3e0f4f72419e9ac71fb3c9817c36 2024-12-10T01:14:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793331180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793331200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da0a89883b4e4fb0a6191deb9f63fa00 is 50, key is test_row_0/C:col10/1733793270340/Put/seqid=0 2024-12-10T01:14:31,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741845_1021 (size=9657) 2024-12-10T01:14:31,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da0a89883b4e4fb0a6191deb9f63fa00 2024-12-10T01:14:31,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/4dc14f59834440afae0a7e38ac344286 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286 2024-12-10T01:14:31,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286, entries=100, sequenceid=60, filesize=9.4 K 2024-12-10T01:14:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/aa2c3e0f4f72419e9ac71fb3c9817c36 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36 2024-12-10T01:14:31,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36, entries=100, sequenceid=60, filesize=9.4 K 2024-12-10T01:14:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da0a89883b4e4fb0a6191deb9f63fa00 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00 2024-12-10T01:14:31,332 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00, entries=100, sequenceid=60, filesize=9.4 K 2024-12-10T01:14:31,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5c85b06919b802618bd08e23219e27af in 684ms, sequenceid=60, compaction requested=true 2024-12-10T01:14:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:31,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-10T01:14:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-10T01:14:31,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T01:14:31,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7280 sec 2024-12-10T01:14:31,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7600 sec 2024-12-10T01:14:31,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793270840, max timestamp=1733793271421Num uniques 1018; 2024-12-10T01:14:31,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:14:31,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793270840, max timestamp=1733793271421Num uniques 1018; 2024-12-10T01:14:31,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:14:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:31,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-10T01:14:31,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:14:31,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7037973b0580413ca801677b58bcae50 is 50, key is test_row_0/A:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:31,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741846_1022 (size=14341) 2024-12-10T01:14:31,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7037973b0580413ca801677b58bcae50 2024-12-10T01:14:31,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/3783f75bd9b94c06ab327b19eb4e1bd6 is 50, key is test_row_0/B:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:31,534 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T01:14:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793331532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793331534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741847_1023 (size=12001) 2024-12-10T01:14:31,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793331544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793331545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793271531, max timestamp=1733793271543Num uniques 859; 2024-12-10T01:14:31,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793271531, max timestamp=1733793271543Num uniques 859; 2024-12-10T01:14:31,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:31,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793331642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793331650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793331650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793331670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793331672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T01:14:31,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:31,761 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793331760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-10T01:14:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:31,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:31,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:31,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:31,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793331860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793331860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:31,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793331880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793331880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:31,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:31,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/3783f75bd9b94c06ab327b19eb4e1bd6 2024-12-10T01:14:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793331969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:31,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddd84712617a41e391a6540a29ada00e is 50, key is test_row_0/C:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:32,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741848_1024 (size=12001) 2024-12-10T01:14:32,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddd84712617a41e391a6540a29ada00e 2024-12-10T01:14:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7037973b0580413ca801677b58bcae50 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50 2024-12-10T01:14:32,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50, entries=200, sequenceid=98, filesize=14.0 K 2024-12-10T01:14:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/3783f75bd9b94c06ab327b19eb4e1bd6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6 2024-12-10T01:14:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:32,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:14:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ddd84712617a41e391a6540a29ada00e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e 2024-12-10T01:14:32,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:14:32,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=118.53 KB/121370 for 5c85b06919b802618bd08e23219e27af in 667ms, sequenceid=98, compaction requested=true 2024-12-10T01:14:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:32,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:32,116 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:32,118 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:32,120 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:32,122 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:32,123 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,124 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=44.6 K 2024-12-10T01:14:32,125 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7838763eac134fc8859d2af6bfb7acfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733793269657 2024-12-10T01:14:32,127 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c22f611cc0ca4ef992829099d29336d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733793269950 2024-12-10T01:14:32,128 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2c3e0f4f72419e9ac71fb3c9817c36, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733793270340 2024-12-10T01:14:32,128 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:32,129 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:32,129 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3783f75bd9b94c06ab327b19eb4e1bd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:32,130 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,136 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=53.7 K 2024-12-10T01:14:32,138 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f377a73693442fd96009980f2d8893e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733793269614 2024-12-10T01:14:32,139 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting feecb7a687714e3789ed3a2cf80b4de4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733793269950 2024-12-10T01:14:32,141 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc14f59834440afae0a7e38ac344286, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733793270340 2024-12-10T01:14:32,143 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7037973b0580413ca801677b58bcae50, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=125.23 KB heapSize=363.39 KB 2024-12-10T01:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:32,192 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:32,195 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/aee16c9799a54fc4a260ab7d0268bd08 is 50, key is test_row_0/B:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:32,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7bff2854907a490d9dd11aff35ee145d is 50, key is test_row_0/A:col10/1733793272178/Put/seqid=0 2024-12-10T01:14:32,220 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#13 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:32,223 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/511c431d5bfd469c8d42f2dbbfb190a8 is 50, key is test_row_0/A:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741849_1025 (size=12139) 2024-12-10T01:14:32,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793332226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793332228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793332231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,245 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/aee16c9799a54fc4a260ab7d0268bd08 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aee16c9799a54fc4a260ab7d0268bd08 2024-12-10T01:14:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793332241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741851_1027 (size=12139) 2024-12-10T01:14:32,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741850_1026 (size=16781) 2024-12-10T01:14:32,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7bff2854907a490d9dd11aff35ee145d 2024-12-10T01:14:32,282 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/511c431d5bfd469c8d42f2dbbfb190a8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/511c431d5bfd469c8d42f2dbbfb190a8 2024-12-10T01:14:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793332289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,299 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into aee16c9799a54fc4a260ab7d0268bd08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:32,299 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:32,302 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=12, startTime=1733793272113; duration=0sec 2024-12-10T01:14:32,303 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:32,303 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:32,303 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:32,305 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 511c431d5bfd469c8d42f2dbbfb190a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:32,305 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:32,305 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=12, startTime=1733793272110; duration=0sec 2024-12-10T01:14:32,305 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:32,305 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:32,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/8d5cdd8327174b8197a535146d793f6f is 50, key is test_row_0/B:col10/1733793272178/Put/seqid=0 2024-12-10T01:14:32,313 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:32,313 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:32,313 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,313 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=44.6 K 2024-12-10T01:14:32,320 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ddae1a27f56c409b93b514c7989685be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733793269657 2024-12-10T01:14:32,321 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f7a8b23ffe4f91989473daeeab1090, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733793269950 2024-12-10T01:14:32,322 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting da0a89883b4e4fb0a6191deb9f63fa00, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733793270340 2024-12-10T01:14:32,324 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd84712617a41e391a6540a29ada00e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:32,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741852_1028 (size=12101) 2024-12-10T01:14:32,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/8d5cdd8327174b8197a535146d793f6f 2024-12-10T01:14:32,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793332350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793332350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793332352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793332350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,357 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#16 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:32,358 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0 is 50, key is test_row_0/C:col10/1733793271421/Put/seqid=0 2024-12-10T01:14:32,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/c5a106275d15481ca5ea057ad6a7d5a3 is 50, key is test_row_0/C:col10/1733793272178/Put/seqid=0 2024-12-10T01:14:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741853_1029 (size=12139) 2024-12-10T01:14:32,387 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:14:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:32,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741854_1030 (size=12101) 2024-12-10T01:14:32,406 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0 2024-12-10T01:14:32,420 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into 90dd42f4126b48dd8d2c7a0f1fa6d4f0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:32,420 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:32,420 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=12, startTime=1733793272113; duration=0sec 2024-12-10T01:14:32,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:32,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:32,477 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T01:14:32,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T01:14:32,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T01:14:32,481 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T01:14:32,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T01:14:32,482 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T01:14:32,484 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T01:14:32,484 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T01:14:32,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793332561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793332561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793332561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793332561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/c5a106275d15481ca5ea057ad6a7d5a3 2024-12-10T01:14:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793332811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/7bff2854907a490d9dd11aff35ee145d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d 2024-12-10T01:14:32,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d, entries=250, sequenceid=130, filesize=16.4 K 2024-12-10T01:14:32,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/8d5cdd8327174b8197a535146d793f6f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f 2024-12-10T01:14:32,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f, entries=150, sequenceid=130, filesize=11.8 K 2024-12-10T01:14:32,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/c5a106275d15481ca5ea057ad6a7d5a3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3 2024-12-10T01:14:32,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3, entries=150, sequenceid=130, filesize=11.8 K 2024-12-10T01:14:32,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:32,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:32,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=53.67 KB/54960 for 5c85b06919b802618bd08e23219e27af in 680ms, sequenceid=130, compaction requested=false 2024-12-10T01:14:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:32,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:32,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:32,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:32,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:32,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/dc10572f69a34fcdb4bbdb9fb0388fc2 is 50, key is test_row_0/A:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741855_1031 (size=12151) 2024-12-10T01:14:32,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793332911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793332919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793332922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793332927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793333030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793333039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793333039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793333040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793333239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793333250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793333251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793333251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/dc10572f69a34fcdb4bbdb9fb0388fc2 2024-12-10T01:14:33,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/be8cf60ec4cd46b1a702b4eb258a28db is 50, key is test_row_0/B:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741856_1032 (size=12151) 2024-12-10T01:14:33,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/be8cf60ec4cd46b1a702b4eb258a28db 2024-12-10T01:14:33,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/8e5e485b62074a4287d8bbd8c335ac55 is 50, key is test_row_0/C:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:33,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741857_1033 (size=12151) 2024-12-10T01:14:33,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793333550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793333559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793333559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793333559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:33,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/8e5e485b62074a4287d8bbd8c335ac55 2024-12-10T01:14:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793333830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:33,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/dc10572f69a34fcdb4bbdb9fb0388fc2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2 2024-12-10T01:14:33,837 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:33,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T01:14:33,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/be8cf60ec4cd46b1a702b4eb258a28db as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db 2024-12-10T01:14:33,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T01:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/8e5e485b62074a4287d8bbd8c335ac55 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55 2024-12-10T01:14:33,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T01:14:33,877 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T01:14:33,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5c85b06919b802618bd08e23219e27af in 1003ms, sequenceid=146, compaction requested=true 2024-12-10T01:14:33,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:33,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:33,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:33,879 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:33,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:33,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:33,880 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:33,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:33,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:33,882 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41071 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:33,882 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:33,882 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:33,882 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:33,882 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,882 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,882 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aee16c9799a54fc4a260ab7d0268bd08, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.5 K 2024-12-10T01:14:33,882 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/511c431d5bfd469c8d42f2dbbfb190a8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=40.1 K 2024-12-10T01:14:33,883 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting aee16c9799a54fc4a260ab7d0268bd08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:33,883 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 511c431d5bfd469c8d42f2dbbfb190a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:33,883 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5cdd8327174b8197a535146d793f6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733793271632 2024-12-10T01:14:33,885 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bff2854907a490d9dd11aff35ee145d, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733793271615 2024-12-10T01:14:33,885 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting be8cf60ec4cd46b1a702b4eb258a28db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:33,886 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc10572f69a34fcdb4bbdb9fb0388fc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:33,911 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:33,912 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/16e6607f04aa4abb9da92ce4884b375c is 50, key is test_row_0/A:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:33,915 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:33,916 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/024e50a35b4f4b9a83e4a124a04e4525 is 50, key is test_row_0/B:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:33,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741858_1034 (size=12391) 2024-12-10T01:14:33,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741859_1035 (size=12391) 2024-12-10T01:14:33,951 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/16e6607f04aa4abb9da92ce4884b375c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/16e6607f04aa4abb9da92ce4884b375c 2024-12-10T01:14:33,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T01:14:33,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,952 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T01:14:33,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:33,965 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 16e6607f04aa4abb9da92ce4884b375c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:33,966 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:33,966 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793273879; duration=0sec 2024-12-10T01:14:33,966 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:33,966 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:33,966 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:33,968 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/024e50a35b4f4b9a83e4a124a04e4525 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/024e50a35b4f4b9a83e4a124a04e4525 2024-12-10T01:14:33,970 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:33,970 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:33,970 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:33,970 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.5 K 2024-12-10T01:14:33,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/07a2cb895e324d7ca89e12a52ba307f9 is 50, key is test_row_0/A:col10/1733793272918/Put/seqid=0 2024-12-10T01:14:33,972 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90dd42f4126b48dd8d2c7a0f1fa6d4f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793270840 2024-12-10T01:14:33,973 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a106275d15481ca5ea057ad6a7d5a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733793271632 2024-12-10T01:14:33,974 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5e485b62074a4287d8bbd8c335ac55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:33,989 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 024e50a35b4f4b9a83e4a124a04e4525(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:33,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:33,990 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793273879; duration=0sec 2024-12-10T01:14:33,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:33,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:34,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741860_1036 (size=12151) 2024-12-10T01:14:34,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/07a2cb895e324d7ca89e12a52ba307f9 2024-12-10T01:14:34,015 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:34,016 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/5d1af5094789478dad222e81d1118ea5 is 50, key is test_row_0/C:col10/1733793272223/Put/seqid=0 2024-12-10T01:14:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/161a45196ad0446994a1d62e688b1491 is 50, key is test_row_0/B:col10/1733793272918/Put/seqid=0 2024-12-10T01:14:34,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741861_1037 (size=12391) 2024-12-10T01:14:34,045 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/5d1af5094789478dad222e81d1118ea5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d1af5094789478dad222e81d1118ea5 2024-12-10T01:14:34,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741862_1038 (size=12151) 2024-12-10T01:14:34,050 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/161a45196ad0446994a1d62e688b1491 2024-12-10T01:14:34,060 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into 5d1af5094789478dad222e81d1118ea5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:34,061 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:34,061 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793273880; duration=0sec 2024-12-10T01:14:34,062 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:34,062 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/9bfd2c8731cf483faa2ee5aa584ceff3 is 50, key is test_row_0/C:col10/1733793272918/Put/seqid=0 2024-12-10T01:14:34,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793334087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793334089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793334091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793334092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741863_1039 (size=12151) 2024-12-10T01:14:34,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793334200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793334200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793334200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793334200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793334410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793334410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793334410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793334410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/9bfd2c8731cf483faa2ee5aa584ceff3 2024-12-10T01:14:34,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/07a2cb895e324d7ca89e12a52ba307f9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9 2024-12-10T01:14:34,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T01:14:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/161a45196ad0446994a1d62e688b1491 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491 2024-12-10T01:14:34,541 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T01:14:34,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/9bfd2c8731cf483faa2ee5aa584ceff3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3 2024-12-10T01:14:34,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T01:14:34,555 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 5c85b06919b802618bd08e23219e27af in 603ms, sequenceid=165, compaction requested=false 2024-12-10T01:14:34,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:34,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:34,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-10T01:14:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-10T01:14:34,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T01:14:34,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7930 sec 2024-12-10T01:14:34,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.8060 sec 2024-12-10T01:14:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:34,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/bd7b1511dde6462797f1978db7affbe9 is 50, key is test_row_0/A:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741864_1040 (size=16931) 2024-12-10T01:14:34,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/bd7b1511dde6462797f1978db7affbe9 2024-12-10T01:14:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793334742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793334743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793334746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/1cb24382946f454fa27959e0e9934702 is 50, key is test_row_0/B:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793334748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741865_1041 (size=12151) 2024-12-10T01:14:34,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/1cb24382946f454fa27959e0e9934702 2024-12-10T01:14:34,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f409c8cf116a41ebaee94984fc4693fb is 50, key is test_row_0/C:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:34,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741866_1042 (size=12151) 2024-12-10T01:14:34,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f409c8cf116a41ebaee94984fc4693fb 2024-12-10T01:14:34,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/bd7b1511dde6462797f1978db7affbe9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9 2024-12-10T01:14:34,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9, entries=250, sequenceid=184, filesize=16.5 K 2024-12-10T01:14:34,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/1cb24382946f454fa27959e0e9934702 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702 2024-12-10T01:14:34,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702, entries=150, sequenceid=184, filesize=11.9 K 2024-12-10T01:14:34,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f409c8cf116a41ebaee94984fc4693fb as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb 2024-12-10T01:14:34,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb, entries=150, sequenceid=184, filesize=11.9 K 2024-12-10T01:14:34,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5c85b06919b802618bd08e23219e27af in 126ms, sequenceid=184, compaction requested=true 2024-12-10T01:14:34,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:34,849 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:34,849 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:34,852 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:34,852 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:34,852 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:34,853 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/024e50a35b4f4b9a83e4a124a04e4525, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.8 K 2024-12-10T01:14:34,854 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:34,854 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:34,854 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:34,854 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/16e6607f04aa4abb9da92ce4884b375c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=40.5 K 2024-12-10T01:14:34,856 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e6607f04aa4abb9da92ce4884b375c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:34,857 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 024e50a35b4f4b9a83e4a124a04e4525, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:34,857 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a2cb895e324d7ca89e12a52ba307f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733793272907 2024-12-10T01:14:34,857 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 161a45196ad0446994a1d62e688b1491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733793272907 2024-12-10T01:14:34,858 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7b1511dde6462797f1978db7affbe9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:34,858 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb24382946f454fa27959e0e9934702, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:34,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:34,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/f869be891aec40b196794da66ae82b13 is 50, key is test_row_0/A:col10/1733793274863/Put/seqid=0 2024-12-10T01:14:34,891 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:34,892 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/ae92434ddd524570a56867fb6331aa9f is 50, key is test_row_0/A:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:34,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793334887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793334888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793334893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,902 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:34,903 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6cbd965718554f9abd3bc63027e32cc7 is 50, key is test_row_0/B:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:34,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793334898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741867_1043 (size=16931) 2024-12-10T01:14:34,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741868_1044 (size=12493) 2024-12-10T01:14:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741869_1045 (size=12493) 2024-12-10T01:14:34,967 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6cbd965718554f9abd3bc63027e32cc7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6cbd965718554f9abd3bc63027e32cc7 2024-12-10T01:14:34,983 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 6cbd965718554f9abd3bc63027e32cc7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:34,983 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:34,983 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793274849; duration=0sec 2024-12-10T01:14:34,984 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:34,984 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:34,984 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:34,985 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:34,985 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:34,985 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:34,986 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d1af5094789478dad222e81d1118ea5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.8 K 2024-12-10T01:14:34,986 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1af5094789478dad222e81d1118ea5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733793272223 2024-12-10T01:14:34,986 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bfd2c8731cf483faa2ee5aa584ceff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733793272907 2024-12-10T01:14:34,987 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f409c8cf116a41ebaee94984fc4693fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793335000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793335000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,006 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:35,006 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/a00c7210e9264fa196fdc5efabbe3c8a is 50, key is test_row_0/C:col10/1733793274084/Put/seqid=0 2024-12-10T01:14:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793335010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793335010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741870_1046 (size=12493) 2024-12-10T01:14:35,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793335211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793335209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793335222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793335222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/f869be891aec40b196794da66ae82b13 2024-12-10T01:14:35,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c60ac497414c492885dd600e4390f839 is 50, key is test_row_0/B:col10/1733793274863/Put/seqid=0 2024-12-10T01:14:35,357 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/ae92434ddd524570a56867fb6331aa9f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ae92434ddd524570a56867fb6331aa9f 2024-12-10T01:14:35,372 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into ae92434ddd524570a56867fb6331aa9f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:35,373 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:35,373 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793274849; duration=0sec 2024-12-10T01:14:35,373 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:35,373 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:35,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741871_1047 (size=12151) 2024-12-10T01:14:35,454 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/a00c7210e9264fa196fdc5efabbe3c8a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/a00c7210e9264fa196fdc5efabbe3c8a 2024-12-10T01:14:35,463 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into a00c7210e9264fa196fdc5efabbe3c8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:35,463 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:35,463 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793274849; duration=0sec 2024-12-10T01:14:35,463 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:35,464 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793335519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793335519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793335529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793335530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c60ac497414c492885dd600e4390f839 2024-12-10T01:14:35,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/5d93bbd3c5ed444f952488844b4dd582 is 50, key is test_row_0/C:col10/1733793274863/Put/seqid=0 2024-12-10T01:14:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741872_1048 (size=12151) 2024-12-10T01:14:35,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/5d93bbd3c5ed444f952488844b4dd582 2024-12-10T01:14:35,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/f869be891aec40b196794da66ae82b13 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13 2024-12-10T01:14:35,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13, entries=250, sequenceid=203, filesize=16.5 K 2024-12-10T01:14:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c60ac497414c492885dd600e4390f839 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839 2024-12-10T01:14:35,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839, entries=150, sequenceid=203, filesize=11.9 K 2024-12-10T01:14:35,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/5d93bbd3c5ed444f952488844b4dd582 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582 2024-12-10T01:14:35,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582, entries=150, sequenceid=203, filesize=11.9 K 2024-12-10T01:14:35,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 5c85b06919b802618bd08e23219e27af in 988ms, sequenceid=203, compaction requested=false 2024-12-10T01:14:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:35,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T01:14:35,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:35,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:35,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:35,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:35,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:35,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:35,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3cfef77fc13045d7a14a615dcb0b94b2 is 50, key is test_row_0/A:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:35,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741873_1049 (size=14541) 2024-12-10T01:14:35,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3cfef77fc13045d7a14a615dcb0b94b2 2024-12-10T01:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T01:14:35,930 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:35,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-10T01:14:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:35,937 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:35,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6834c4d9d5ae4890a75fb32674b71895 is 50, key is test_row_0/B:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:35,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:35,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:35,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793335944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:35,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741874_1050 (size=12151) 2024-12-10T01:14:35,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6834c4d9d5ae4890a75fb32674b71895 2024-12-10T01:14:35,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/102204dbf5744cc2accaa15337e9e372 is 50, key is test_row_0/C:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:35,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741875_1051 (size=12151) 2024-12-10T01:14:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793336029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:36,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793336040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793336040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793336040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793336060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T01:14:36,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:36,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T01:14:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793336269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/102204dbf5744cc2accaa15337e9e372 2024-12-10T01:14:36,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T01:14:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:36,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3cfef77fc13045d7a14a615dcb0b94b2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2 2024-12-10T01:14:36,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2, entries=200, sequenceid=221, filesize=14.2 K 2024-12-10T01:14:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6834c4d9d5ae4890a75fb32674b71895 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895 2024-12-10T01:14:36,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T01:14:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/102204dbf5744cc2accaa15337e9e372 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372 2024-12-10T01:14:36,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T01:14:36,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5c85b06919b802618bd08e23219e27af in 574ms, sequenceid=221, compaction requested=true 2024-12-10T01:14:36,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:36,442 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:36,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:36,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:36,443 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:36,444 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:36,444 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:36,444 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:36,444 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:36,444 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,444 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,444 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6cbd965718554f9abd3bc63027e32cc7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.9 K 2024-12-10T01:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:36,444 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ae92434ddd524570a56867fb6331aa9f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=42.9 K 2024-12-10T01:14:36,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:36,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:36,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:36,445 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae92434ddd524570a56867fb6331aa9f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:36,445 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbd965718554f9abd3bc63027e32cc7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:36,446 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c60ac497414c492885dd600e4390f839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733793274745 2024-12-10T01:14:36,446 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f869be891aec40b196794da66ae82b13, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733793274741 2024-12-10T01:14:36,446 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfef77fc13045d7a14a615dcb0b94b2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:36,447 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6834c4d9d5ae4890a75fb32674b71895, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:36,460 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:36,461 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:36,461 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/0cc3c1e353af4dfc99d8fa2e550d20c9 is 50, key is test_row_0/A:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:36,461 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6c6e10c2fd694be19cd4021a07d3a895 is 50, key is test_row_0/B:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741876_1052 (size=12595) 2024-12-10T01:14:36,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741877_1053 (size=12595) 2024-12-10T01:14:36,486 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/0cc3c1e353af4dfc99d8fa2e550d20c9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/0cc3c1e353af4dfc99d8fa2e550d20c9 2024-12-10T01:14:36,487 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/6c6e10c2fd694be19cd4021a07d3a895 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6c6e10c2fd694be19cd4021a07d3a895 2024-12-10T01:14:36,498 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 0cc3c1e353af4dfc99d8fa2e550d20c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:36,498 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 6c6e10c2fd694be19cd4021a07d3a895(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:36,498 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793276442; duration=0sec 2024-12-10T01:14:36,498 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793276443; duration=0sec 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:36,498 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:36,499 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:36,500 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:36,500 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:36,500 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,500 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/a00c7210e9264fa196fdc5efabbe3c8a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=35.9 K 2024-12-10T01:14:36,500 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00c7210e9264fa196fdc5efabbe3c8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733793274084 2024-12-10T01:14:36,501 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d93bbd3c5ed444f952488844b4dd582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733793274745 2024-12-10T01:14:36,502 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102204dbf5744cc2accaa15337e9e372, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:36,517 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:36,518 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bb05e48e1a6b46deb2189ed099a708d5 is 50, key is test_row_0/C:col10/1733793275865/Put/seqid=0 2024-12-10T01:14:36,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741878_1054 (size=12595) 2024-12-10T01:14:36,543 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bb05e48e1a6b46deb2189ed099a708d5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bb05e48e1a6b46deb2189ed099a708d5 2024-12-10T01:14:36,554 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into bb05e48e1a6b46deb2189ed099a708d5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:36,554 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:36,554 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793276445; duration=0sec 2024-12-10T01:14:36,554 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:36,554 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:36,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T01:14:36,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:36,556 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T01:14:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:36,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:36,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:36,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/502a145b61fe4293b2aebdfe18eb8a63 is 50, key is test_row_0/A:col10/1733793275939/Put/seqid=0 2024-12-10T01:14:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:36,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741879_1055 (size=12151) 2024-12-10T01:14:36,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793336639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793336749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793336960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:36,989 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/502a145b61fe4293b2aebdfe18eb8a63 2024-12-10T01:14:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793337040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,042 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/ffc1dee74ded4f99b577b5d2787bde28 is 50, key is test_row_0/B:col10/1733793275939/Put/seqid=0 2024-12-10T01:14:37,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741880_1056 (size=12151) 2024-12-10T01:14:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793337061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793337060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793337061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,063 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,063 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,064 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793337270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/ffc1dee74ded4f99b577b5d2787bde28 2024-12-10T01:14:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f95b7640a2fc4419b9aecc401bc7a7f4 is 50, key is test_row_0/C:col10/1733793275939/Put/seqid=0 2024-12-10T01:14:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741881_1057 (size=12151) 2024-12-10T01:14:37,480 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f95b7640a2fc4419b9aecc401bc7a7f4 2024-12-10T01:14:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/502a145b61fe4293b2aebdfe18eb8a63 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63 2024-12-10T01:14:37,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T01:14:37,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/ffc1dee74ded4f99b577b5d2787bde28 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28 2024-12-10T01:14:37,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T01:14:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/f95b7640a2fc4419b9aecc401bc7a7f4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4 2024-12-10T01:14:37,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T01:14:37,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5c85b06919b802618bd08e23219e27af in 965ms, sequenceid=241, compaction requested=false 2024-12-10T01:14:37,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:37,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:37,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-10T01:14:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-10T01:14:37,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-10T01:14:37,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-10T01:14:37,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5960 sec 2024-12-10T01:14:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:37,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:37,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae is 50, key is test_row_0/A:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:37,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741882_1058 (size=14591) 2024-12-10T01:14:37,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793337846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793337959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T01:14:38,080 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:38,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-10T01:14:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:38,083 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:38,084 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:38,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793338169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:38,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae 2024-12-10T01:14:38,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5e6eda91485e4a50a5c9baae5b62cff8 is 50, key is test_row_0/B:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741883_1059 (size=12201) 2024-12-10T01:14:38,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793338479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:38,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5e6eda91485e4a50a5c9baae5b62cff8 2024-12-10T01:14:38,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/93e41ca32c64448c8a4a3a6051f0b6dc is 50, key is test_row_0/C:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741884_1060 (size=12201) 2024-12-10T01:14:38,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:38,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:38,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:38,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793338990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/93e41ca32c64448c8a4a3a6051f0b6dc 2024-12-10T01:14:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae 2024-12-10T01:14:39,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae, entries=200, sequenceid=257, filesize=14.2 K 2024-12-10T01:14:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5e6eda91485e4a50a5c9baae5b62cff8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8 2024-12-10T01:14:39,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8, entries=150, sequenceid=257, filesize=11.9 K 2024-12-10T01:14:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/93e41ca32c64448c8a4a3a6051f0b6dc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc 2024-12-10T01:14:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793339060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,063 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc, entries=150, sequenceid=257, filesize=11.9 K 2024-12-10T01:14:39,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5c85b06919b802618bd08e23219e27af in 1290ms, sequenceid=257, compaction requested=true 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:39,073 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:39,073 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:39,074 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:39,074 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:39,075 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,075 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6c6e10c2fd694be19cd4021a07d3a895, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.1 K 2024-12-10T01:14:39,075 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39337 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:39,075 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:39,075 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,075 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/0cc3c1e353af4dfc99d8fa2e550d20c9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=38.4 K 2024-12-10T01:14:39,076 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc3c1e353af4dfc99d8fa2e550d20c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:39,076 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6e10c2fd694be19cd4021a07d3a895, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:39,077 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 502a145b61fe4293b2aebdfe18eb8a63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733793275927 2024-12-10T01:14:39,077 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc1dee74ded4f99b577b5d2787bde28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733793275927 2024-12-10T01:14:39,077 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd3a8ce1f5e481c95e5f7c3cebf27ae, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276619 2024-12-10T01:14:39,077 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6eda91485e4a50a5c9baae5b62cff8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276626 2024-12-10T01:14:39,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T01:14:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:39,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9191d8d6cfa044b8a747bd433ba7f041 is 50, key is test_row_0/A:col10/1733793277838/Put/seqid=0 2024-12-10T01:14:39,107 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:39,108 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/45662c69755747fe900e257fb7c5ea5e is 50, key is test_row_0/A:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793339114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793339120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793339120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,128 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:39,129 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/99454f4233804a22988667512e74e830 is 50, key is test_row_0/B:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741885_1061 (size=12301) 2024-12-10T01:14:39,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9191d8d6cfa044b8a747bd433ba7f041 2024-12-10T01:14:39,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741887_1063 (size=12747) 2024-12-10T01:14:39,177 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/99454f4233804a22988667512e74e830 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/99454f4233804a22988667512e74e830 2024-12-10T01:14:39,192 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 99454f4233804a22988667512e74e830(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:39,192 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:39,192 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793279073; duration=0sec 2024-12-10T01:14:39,193 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:39,193 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:39,193 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741886_1062 (size=12747) 2024-12-10T01:14:39,198 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:39,198 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:39,198 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,198 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bb05e48e1a6b46deb2189ed099a708d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.1 K 2024-12-10T01:14:39,201 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting bb05e48e1a6b46deb2189ed099a708d5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733793274877 2024-12-10T01:14:39,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0695176466ba4768959e591f1d4efc31 is 50, key is test_row_0/B:col10/1733793277838/Put/seqid=0 2024-12-10T01:14:39,202 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f95b7640a2fc4419b9aecc401bc7a7f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733793275927 2024-12-10T01:14:39,203 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e41ca32c64448c8a4a3a6051f0b6dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276626 2024-12-10T01:14:39,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741888_1064 (size=12301) 2024-12-10T01:14:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:39,221 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:39,221 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/721766ba80714d24b47fba8c1f0d3d58 is 50, key is test_row_0/C:col10/1733793277781/Put/seqid=0 2024-12-10T01:14:39,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793339230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793339230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793339237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741889_1065 (size=12747) 2024-12-10T01:14:39,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793339440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793339450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793339450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,601 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/45662c69755747fe900e257fb7c5ea5e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/45662c69755747fe900e257fb7c5ea5e 2024-12-10T01:14:39,608 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 45662c69755747fe900e257fb7c5ea5e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:39,608 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:39,608 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793279073; duration=0sec 2024-12-10T01:14:39,608 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:39,608 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:39,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0695176466ba4768959e591f1d4efc31 2024-12-10T01:14:39,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/d351647b18974420919080c49ec3b736 is 50, key is test_row_0/C:col10/1733793277838/Put/seqid=0 2024-12-10T01:14:39,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741890_1066 (size=12301) 2024-12-10T01:14:39,656 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/721766ba80714d24b47fba8c1f0d3d58 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/721766ba80714d24b47fba8c1f0d3d58 2024-12-10T01:14:39,663 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into 721766ba80714d24b47fba8c1f0d3d58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:39,663 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:39,663 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793279073; duration=0sec 2024-12-10T01:14:39,664 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:39,664 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:39,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793339750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793339759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793339759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:39,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:39,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:40,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793339999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,004 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/d351647b18974420919080c49ec3b736 2024-12-10T01:14:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9191d8d6cfa044b8a747bd433ba7f041 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041 2024-12-10T01:14:40,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T01:14:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0695176466ba4768959e591f1d4efc31 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31 2024-12-10T01:14:40,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T01:14:40,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/d351647b18974420919080c49ec3b736 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736 2024-12-10T01:14:40,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T01:14:40,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5c85b06919b802618bd08e23219e27af in 995ms, sequenceid=274, compaction requested=false 2024-12-10T01:14:40,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:40,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T01:14:40,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:40,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T01:14:40,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:40,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:40,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3e2c873168e24469b85f0070c0a2f7b4 is 50, key is test_row_0/A:col10/1733793279091/Put/seqid=0 2024-12-10T01:14:40,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741891_1067 (size=12301) 2024-12-10T01:14:40,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3e2c873168e24469b85f0070c0a2f7b4 2024-12-10T01:14:40,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c64922d9aad24fd3ad2df9c781df7506 is 50, key is test_row_0/B:col10/1733793279091/Put/seqid=0 2024-12-10T01:14:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741892_1068 (size=12301) 2024-12-10T01:14:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:40,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793340290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793340296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793340297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793340399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793340410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793340410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c64922d9aad24fd3ad2df9c781df7506 2024-12-10T01:14:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/061b0448dfd24e49a91d45e895aff16a is 50, key is test_row_0/C:col10/1733793279091/Put/seqid=0 2024-12-10T01:14:40,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741893_1069 (size=12301) 2024-12-10T01:14:40,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793340609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793340619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793340620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793340919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793340929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793340930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:40,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/061b0448dfd24e49a91d45e895aff16a 2024-12-10T01:14:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3e2c873168e24469b85f0070c0a2f7b4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4 2024-12-10T01:14:40,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4, entries=150, sequenceid=292, filesize=12.0 K 2024-12-10T01:14:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c64922d9aad24fd3ad2df9c781df7506 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506 2024-12-10T01:14:40,971 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506, entries=150, sequenceid=292, filesize=12.0 K 2024-12-10T01:14:40,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/061b0448dfd24e49a91d45e895aff16a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a 2024-12-10T01:14:40,978 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a, entries=150, sequenceid=292, filesize=12.0 K 2024-12-10T01:14:40,979 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 5c85b06919b802618bd08e23219e27af in 887ms, sequenceid=292, compaction requested=true 2024-12-10T01:14:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-10T01:14:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-10T01:14:40,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T01:14:40,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8970 sec 2024-12-10T01:14:40,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.9040 sec 2024-12-10T01:14:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:41,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T01:14:41,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:41,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:41,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:41,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:41,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:41,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:41,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/98ac2f78601c4e60b31937e9092e9861 is 50, key is test_row_0/A:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793341473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793341475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741894_1070 (size=14741) 2024-12-10T01:14:41,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/98ac2f78601c4e60b31937e9092e9861 2024-12-10T01:14:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793341477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/e94dc6b34bc648148f998c10b009769c is 50, key is test_row_0/B:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:41,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741895_1071 (size=12301) 2024-12-10T01:14:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793341590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793341591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793341600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:41,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/e94dc6b34bc648148f998c10b009769c 2024-12-10T01:14:41,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bcc0ed287e82472baefee61039e7f72c is 50, key is test_row_0/C:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:41,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741896_1072 (size=12301) 2024-12-10T01:14:41,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bcc0ed287e82472baefee61039e7f72c 2024-12-10T01:14:41,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/98ac2f78601c4e60b31937e9092e9861 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861 2024-12-10T01:14:41,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861, entries=200, sequenceid=311, filesize=14.4 K 2024-12-10T01:14:41,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/e94dc6b34bc648148f998c10b009769c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c 2024-12-10T01:14:41,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T01:14:41,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bcc0ed287e82472baefee61039e7f72c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c 2024-12-10T01:14:41,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T01:14:41,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5c85b06919b802618bd08e23219e27af in 561ms, sequenceid=311, compaction requested=true 2024-12-10T01:14:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:41,997 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:41,997 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:41,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:41,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:41,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:42,000 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:42,000 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:42,000 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:42,000 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:42,000 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,000 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,000 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/99454f4233804a22988667512e74e830, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=48.5 K 2024-12-10T01:14:42,000 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/45662c69755747fe900e257fb7c5ea5e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=50.9 K 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 99454f4233804a22988667512e74e830, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276626 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45662c69755747fe900e257fb7c5ea5e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276626 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0695176466ba4768959e591f1d4efc31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733793277838 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9191d8d6cfa044b8a747bd433ba7f041, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733793277838 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c64922d9aad24fd3ad2df9c781df7506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733793279091 2024-12-10T01:14:42,001 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2c873168e24469b85f0070c0a2f7b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733793279091 2024-12-10T01:14:42,002 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e94dc6b34bc648148f998c10b009769c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:42,002 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ac2f78601c4e60b31937e9092e9861, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:42,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T01:14:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,039 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:42,040 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:42,040 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/10d7ac268fc7426988d01780056b5451 is 50, key is test_row_0/A:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:42,040 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/21f95c0094e64886aadcd85d28b84ed2 is 50, key is test_row_0/B:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:42,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e5ceb2a8010b485c962acf0399dbfe00 is 50, key is test_row_0/A:col10/1733793282029/Put/seqid=0 2024-12-10T01:14:42,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741897_1073 (size=12983) 2024-12-10T01:14:42,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741898_1074 (size=12983) 2024-12-10T01:14:42,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741899_1075 (size=14741) 2024-12-10T01:14:42,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e5ceb2a8010b485c962acf0399dbfe00 2024-12-10T01:14:42,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/65084c056c8645ee97c9330f3c72b9db is 50, key is test_row_0/B:col10/1733793282029/Put/seqid=0 2024-12-10T01:14:42,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741900_1076 (size=12301) 2024-12-10T01:14:42,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793342119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793342119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793342119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793342120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793342229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T01:14:42,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:42,250 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-10T01:14:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:42,253 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:42,254 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:42,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793342439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,465 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/10d7ac268fc7426988d01780056b5451 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/10d7ac268fc7426988d01780056b5451 2024-12-10T01:14:42,473 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/21f95c0094e64886aadcd85d28b84ed2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/21f95c0094e64886aadcd85d28b84ed2 2024-12-10T01:14:42,473 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 10d7ac268fc7426988d01780056b5451(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:42,473 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:42,473 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=12, startTime=1733793281997; duration=0sec 2024-12-10T01:14:42,473 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:42,473 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:42,473 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:42,475 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:42,475 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:42,475 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,475 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/721766ba80714d24b47fba8c1f0d3d58, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=48.5 K 2024-12-10T01:14:42,476 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721766ba80714d24b47fba8c1f0d3d58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733793276626 2024-12-10T01:14:42,476 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d351647b18974420919080c49ec3b736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733793277838 2024-12-10T01:14:42,476 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 061b0448dfd24e49a91d45e895aff16a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733793279091 2024-12-10T01:14:42,477 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc0ed287e82472baefee61039e7f72c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:42,479 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 21f95c0094e64886aadcd85d28b84ed2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:42,479 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:42,479 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=12, startTime=1733793281997; duration=0sec 2024-12-10T01:14:42,479 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:42,479 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:42,489 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:42,490 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8 is 50, key is test_row_0/C:col10/1733793280286/Put/seqid=0 2024-12-10T01:14:42,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/65084c056c8645ee97c9330f3c72b9db 2024-12-10T01:14:42,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741901_1077 (size=12983) 2024-12-10T01:14:42,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/959298279d014560b8c918bd90e61b4b is 50, key is test_row_0/C:col10/1733793282029/Put/seqid=0 2024-12-10T01:14:42,511 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8 2024-12-10T01:14:42,520 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into e6f55786fb4f4f64bcc1ffdf31f5ddc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:42,520 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:42,520 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=12, startTime=1733793281998; duration=0sec 2024-12-10T01:14:42,520 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:42,520 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741902_1078 (size=12301) 2024-12-10T01:14:42,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/959298279d014560b8c918bd90e61b4b 2024-12-10T01:14:42,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e5ceb2a8010b485c962acf0399dbfe00 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00 2024-12-10T01:14:42,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00, entries=200, sequenceid=325, filesize=14.4 K 2024-12-10T01:14:42,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/65084c056c8645ee97c9330f3c72b9db as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db 2024-12-10T01:14:42,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:42,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db, entries=150, sequenceid=325, filesize=12.0 K 2024-12-10T01:14:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/959298279d014560b8c918bd90e61b4b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b 2024-12-10T01:14:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:42,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b, entries=150, sequenceid=325, filesize=12.0 K 2024-12-10T01:14:42,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5c85b06919b802618bd08e23219e27af in 540ms, sequenceid=325, compaction requested=false 2024-12-10T01:14:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:42,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:42,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/d4e2ece3c38045e2a22a55c4cdc75803 is 50, key is test_row_0/A:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:42,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741903_1079 (size=17181) 2024-12-10T01:14:42,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/d4e2ece3c38045e2a22a55c4cdc75803 2024-12-10T01:14:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793342646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793342648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793342648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/84b7b0a43352453a830f4d1e7926f7ae is 50, key is test_row_0/B:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:42,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741904_1080 (size=12301) 2024-12-10T01:14:42,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793342749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793342759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793342759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793342759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:42,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793342969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793342970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793342970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/84b7b0a43352453a830f4d1e7926f7ae 2024-12-10T01:14:43,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/48a84216ce264d69943780b071b3f2c4 is 50, key is test_row_0/C:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:43,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741905_1081 (size=12301) 2024-12-10T01:14:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39850 deadline: 1733793343110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,112 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793343259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793343279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793343280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793343280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:43,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:43,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/48a84216ce264d69943780b071b3f2c4 2024-12-10T01:14:43,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/d4e2ece3c38045e2a22a55c4cdc75803 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803 2024-12-10T01:14:43,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803, entries=250, sequenceid=348, filesize=16.8 K 2024-12-10T01:14:43,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/84b7b0a43352453a830f4d1e7926f7ae as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae 2024-12-10T01:14:43,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae, entries=150, sequenceid=348, filesize=12.0 K 2024-12-10T01:14:43,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/48a84216ce264d69943780b071b3f2c4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4 2024-12-10T01:14:43,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4, entries=150, sequenceid=348, filesize=12.0 K 2024-12-10T01:14:43,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5c85b06919b802618bd08e23219e27af in 884ms, sequenceid=348, compaction requested=true 2024-12-10T01:14:43,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:43,516 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:43,516 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:43,517 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:43,517 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:43,517 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,517 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/10d7ac268fc7426988d01780056b5451, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=43.9 K 2024-12-10T01:14:43,517 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:43,517 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:43,517 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,517 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/21f95c0094e64886aadcd85d28b84ed2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.7 K 2024-12-10T01:14:43,517 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d7ac268fc7426988d01780056b5451, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:43,518 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ceb2a8010b485c962acf0399dbfe00, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733793281454 2024-12-10T01:14:43,518 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f95c0094e64886aadcd85d28b84ed2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:43,518 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e2ece3c38045e2a22a55c4cdc75803, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282108 2024-12-10T01:14:43,518 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 65084c056c8645ee97c9330f3c72b9db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733793281454 2024-12-10T01:14:43,518 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b7b0a43352453a830f4d1e7926f7ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282115 2024-12-10T01:14:43,531 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:43,531 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3a2b1a2a12b2463d841e9a82ca6e48fb is 50, key is test_row_0/A:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:43,535 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:43,535 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/4528fbf37d7e409ab7379b4a8978bcf9 is 50, key is test_row_0/B:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:43,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741906_1082 (size=13085) 2024-12-10T01:14:43,549 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3a2b1a2a12b2463d841e9a82ca6e48fb as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3a2b1a2a12b2463d841e9a82ca6e48fb 2024-12-10T01:14:43,556 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 3a2b1a2a12b2463d841e9a82ca6e48fb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:43,557 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:43,557 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793283515; duration=0sec 2024-12-10T01:14:43,557 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:43,557 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:43,557 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:43,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741907_1083 (size=13085) 2024-12-10T01:14:43,560 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:43,560 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:43,560 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,560 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.7 K 2024-12-10T01:14:43,561 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f55786fb4f4f64bcc1ffdf31f5ddc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733793280277 2024-12-10T01:14:43,561 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959298279d014560b8c918bd90e61b4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733793281454 2024-12-10T01:14:43,563 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a84216ce264d69943780b071b3f2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282115 2024-12-10T01:14:43,569 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/4528fbf37d7e409ab7379b4a8978bcf9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/4528fbf37d7e409ab7379b4a8978bcf9 2024-12-10T01:14:43,576 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:43,577 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/1d506d46b24c4a7aa2e309efe1a67c03 is 50, key is test_row_0/C:col10/1733793282630/Put/seqid=0 2024-12-10T01:14:43,577 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 4528fbf37d7e409ab7379b4a8978bcf9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:43,577 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:43,577 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793283516; duration=0sec 2024-12-10T01:14:43,577 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:43,577 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:43,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741908_1084 (size=13085) 2024-12-10T01:14:43,586 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/1d506d46b24c4a7aa2e309efe1a67c03 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/1d506d46b24c4a7aa2e309efe1a67c03 2024-12-10T01:14:43,595 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into 1d506d46b24c4a7aa2e309efe1a67c03(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:43,595 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:43,595 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793283516; duration=0sec 2024-12-10T01:14:43,595 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:43,595 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:43,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T01:14:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:43,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:14:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:43,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:43,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:43,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e7262da91ee94045ad353d3c64d708f4 is 50, key is test_row_0/A:col10/1733793282645/Put/seqid=0 2024-12-10T01:14:43,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741909_1085 (size=12301) 2024-12-10T01:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:43,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793343819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793343820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793343820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793343930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793343953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793343954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e7262da91ee94045ad353d3c64d708f4 2024-12-10T01:14:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c12046c32ade4b31a31f93e00d7997d6 is 50, key is test_row_0/B:col10/1733793282645/Put/seqid=0 2024-12-10T01:14:44,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741910_1086 (size=12301) 2024-12-10T01:14:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793344139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793344160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793344160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793344269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,272 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793344450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,458 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c12046c32ade4b31a31f93e00d7997d6 2024-12-10T01:14:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/abc5b53b4c5d437e8387552dbf11a9fc is 50, key is test_row_0/C:col10/1733793282645/Put/seqid=0 2024-12-10T01:14:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793344469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793344469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741911_1087 (size=12301) 2024-12-10T01:14:44,478 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/abc5b53b4c5d437e8387552dbf11a9fc 2024-12-10T01:14:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/e7262da91ee94045ad353d3c64d708f4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4 2024-12-10T01:14:44,489 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T01:14:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c12046c32ade4b31a31f93e00d7997d6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6 2024-12-10T01:14:44,497 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T01:14:44,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/abc5b53b4c5d437e8387552dbf11a9fc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc 2024-12-10T01:14:44,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T01:14:44,505 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5c85b06919b802618bd08e23219e27af in 870ms, sequenceid=364, compaction requested=false 2024-12-10T01:14:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-10T01:14:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-10T01:14:44,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-10T01:14:44,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2530 sec 2024-12-10T01:14:44,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2600 sec 2024-12-10T01:14:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:44,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:44,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/fa34a0fcc93f4b27b5addf8fbe695824 is 50, key is test_row_0/A:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:44,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741912_1088 (size=12301) 2024-12-10T01:14:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793344984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793344984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793344985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793345089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793345099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793345099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793345299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793345309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793345309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/fa34a0fcc93f4b27b5addf8fbe695824 2024-12-10T01:14:45,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/18c5787c0d3046fd876b0b339c9ed2c8 is 50, key is test_row_0/B:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:45,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741913_1089 (size=12301) 2024-12-10T01:14:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793345610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793345619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793345619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:45,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/18c5787c0d3046fd876b0b339c9ed2c8 2024-12-10T01:14:45,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/efb7eeffe5f444bab5b619e3d16a77cf is 50, key is test_row_0/C:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:45,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741914_1090 (size=12301) 2024-12-10T01:14:46,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793346119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793346129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793346129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/efb7eeffe5f444bab5b619e3d16a77cf 2024-12-10T01:14:46,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/fa34a0fcc93f4b27b5addf8fbe695824 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824 2024-12-10T01:14:46,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824, entries=150, sequenceid=385, filesize=12.0 K 2024-12-10T01:14:46,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/18c5787c0d3046fd876b0b339c9ed2c8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8 2024-12-10T01:14:46,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8, entries=150, sequenceid=385, filesize=12.0 K 2024-12-10T01:14:46,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/efb7eeffe5f444bab5b619e3d16a77cf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf 2024-12-10T01:14:46,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf, entries=150, sequenceid=385, filesize=12.0 K 2024-12-10T01:14:46,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5c85b06919b802618bd08e23219e27af in 1274ms, sequenceid=385, compaction requested=true 2024-12-10T01:14:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:46,236 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:46,236 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:46,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:46,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:46,237 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:46,237 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:46,238 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:46,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:46,238 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,238 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,238 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/4528fbf37d7e409ab7379b4a8978bcf9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.8 K 2024-12-10T01:14:46,238 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3a2b1a2a12b2463d841e9a82ca6e48fb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.8 K 2024-12-10T01:14:46,238 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a2b1a2a12b2463d841e9a82ca6e48fb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282115 2024-12-10T01:14:46,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4528fbf37d7e409ab7379b4a8978bcf9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282115 2024-12-10T01:14:46,239 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c12046c32ade4b31a31f93e00d7997d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733793282644 2024-12-10T01:14:46,239 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7262da91ee94045ad353d3c64d708f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733793282644 2024-12-10T01:14:46,239 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c5787c0d3046fd876b0b339c9ed2c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:46,239 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa34a0fcc93f4b27b5addf8fbe695824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:46,246 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#79 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:46,247 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:46,247 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/d7bfb9726b90493b84478ac55155af67 is 50, key is test_row_0/A:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:46,247 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/1bcb301cc3af4922b4bcd2698122cefd is 50, key is test_row_0/B:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:46,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741916_1092 (size=13187) 2024-12-10T01:14:46,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741915_1091 (size=13187) 2024-12-10T01:14:46,259 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/1bcb301cc3af4922b4bcd2698122cefd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1bcb301cc3af4922b4bcd2698122cefd 2024-12-10T01:14:46,267 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 1bcb301cc3af4922b4bcd2698122cefd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:46,267 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:46,267 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=13, startTime=1733793286236; duration=0sec 2024-12-10T01:14:46,267 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:46,268 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:46,268 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:14:46,269 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:14:46,269 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:46,269 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,269 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/1d506d46b24c4a7aa2e309efe1a67c03, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=36.8 K 2024-12-10T01:14:46,270 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d506d46b24c4a7aa2e309efe1a67c03, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733793282115 2024-12-10T01:14:46,270 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting abc5b53b4c5d437e8387552dbf11a9fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733793282644 2024-12-10T01:14:46,271 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting efb7eeffe5f444bab5b619e3d16a77cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:46,278 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#80 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:46,279 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/fdad45cce32742efba8f091a4642ccb8 is 50, key is test_row_0/C:col10/1733793283814/Put/seqid=0 2024-12-10T01:14:46,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741917_1093 (size=13187) 2024-12-10T01:14:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:46,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:46,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9b8b8959627b41f78cca7f1738b5086a is 50, key is test_row_0/A:col10/1733793284983/Put/seqid=0 2024-12-10T01:14:46,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741918_1094 (size=12301) 2024-12-10T01:14:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793346366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T01:14:46,429 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:46,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-10T01:14:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:46,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:46,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:46,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793346480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:46,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,660 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/d7bfb9726b90493b84478ac55155af67 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d7bfb9726b90493b84478ac55155af67 2024-12-10T01:14:46,670 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into d7bfb9726b90493b84478ac55155af67(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:46,670 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:46,670 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=13, startTime=1733793286236; duration=0sec 2024-12-10T01:14:46,670 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:46,670 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:46,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793346689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:46,694 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/fdad45cce32742efba8f091a4642ccb8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/fdad45cce32742efba8f091a4642ccb8 2024-12-10T01:14:46,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9b8b8959627b41f78cca7f1738b5086a 2024-12-10T01:14:46,708 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into fdad45cce32742efba8f091a4642ccb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:46,709 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:46,709 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=13, startTime=1733793286236; duration=0sec 2024-12-10T01:14:46,709 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:46,709 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:46,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0ee4fea4d72841448932ba548ebc023a is 50, key is test_row_0/B:col10/1733793284983/Put/seqid=0 2024-12-10T01:14:46,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741919_1095 (size=12301) 2024-12-10T01:14:46,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:46,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:46,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793346999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:47,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:47,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793347139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793347140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:47,140 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:47,141 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:47,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0ee4fea4d72841448932ba548ebc023a 2024-12-10T01:14:47,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793347149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:47,150 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5c85b06919b802618bd08e23219e27af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:14:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:47,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ade1eaebb456402e85ba0a9717aad855 is 50, key is test_row_0/C:col10/1733793284983/Put/seqid=0 2024-12-10T01:14:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741920_1096 (size=12301) 2024-12-10T01:14:47,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:47,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793347510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:47,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ade1eaebb456402e85ba0a9717aad855 2024-12-10T01:14:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/9b8b8959627b41f78cca7f1738b5086a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a 2024-12-10T01:14:47,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a, entries=150, sequenceid=398, filesize=12.0 K 2024-12-10T01:14:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/0ee4fea4d72841448932ba548ebc023a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a 2024-12-10T01:14:47,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a, entries=150, sequenceid=398, filesize=12.0 K 2024-12-10T01:14:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/ade1eaebb456402e85ba0a9717aad855 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855 2024-12-10T01:14:47,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855, entries=150, sequenceid=398, filesize=12.0 K 2024-12-10T01:14:47,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5c85b06919b802618bd08e23219e27af in 1306ms, sequenceid=398, compaction requested=false 2024-12-10T01:14:47,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:47,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T01:14:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:47,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:14:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:47,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/8f49a39091204baeaf5176bbb45d7e2a is 50, key is test_row_0/A:col10/1733793286358/Put/seqid=0 2024-12-10T01:14:47,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741921_1097 (size=12301) 2024-12-10T01:14:48,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/8f49a39091204baeaf5176bbb45d7e2a 2024-12-10T01:14:48,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5916f752a2304b49bef3bd3c36dd7a42 is 50, key is test_row_0/B:col10/1733793286358/Put/seqid=0 2024-12-10T01:14:48,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741922_1098 (size=12301) 2024-12-10T01:14:48,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5916f752a2304b49bef3bd3c36dd7a42 2024-12-10T01:14:48,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/29c46dd3283f4d53b467609c2d1c8879 is 50, key is test_row_0/C:col10/1733793286358/Put/seqid=0 2024-12-10T01:14:48,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741923_1099 (size=12301) 2024-12-10T01:14:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:48,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. as already flushing 2024-12-10T01:14:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793348560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:48,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793348677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:48,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793348889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:48,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/29c46dd3283f4d53b467609c2d1c8879 2024-12-10T01:14:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/8f49a39091204baeaf5176bbb45d7e2a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a 2024-12-10T01:14:48,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a, entries=150, sequenceid=420, filesize=12.0 K 2024-12-10T01:14:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/5916f752a2304b49bef3bd3c36dd7a42 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42 2024-12-10T01:14:48,957 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42, entries=150, sequenceid=420, filesize=12.0 K 2024-12-10T01:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/29c46dd3283f4d53b467609c2d1c8879 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879 2024-12-10T01:14:48,964 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879, entries=150, sequenceid=420, filesize=12.0 K 2024-12-10T01:14:48,965 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5c85b06919b802618bd08e23219e27af in 1308ms, sequenceid=420, compaction requested=true 2024-12-10T01:14:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-10T01:14:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-10T01:14:48,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-10T01:14:48,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-12-10T01:14:48,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5400 sec 2024-12-10T01:14:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:49,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:49,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/63e13defb4a9486b9aa054e12580f600 is 50, key is test_row_0/A:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:49,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741924_1100 (size=14741) 2024-12-10T01:14:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793349188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793349189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793349189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793349199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793349299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793349299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793349299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793349509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793349509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793349509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/63e13defb4a9486b9aa054e12580f600 2024-12-10T01:14:49,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/f7705e8eba9d4af085045d5da6f886aa is 50, key is test_row_0/B:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:49,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741925_1101 (size=12301) 2024-12-10T01:14:49,596 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-10T01:14:49,596 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:49,597 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-10T01:14:49,597 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:49,599 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-10T01:14:49,599 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:49,602 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:49,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:49,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:49,604 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-10T01:14:49,604 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:49,604 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,604 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:49,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39770 deadline: 1733793349709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793349820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793349820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793349820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:49,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/f7705e8eba9d4af085045d5da6f886aa 2024-12-10T01:14:49,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/99880bc547e344e7b7072c9246a5a3f8 is 50, key is test_row_0/C:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:49,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741926_1102 (size=12301) 2024-12-10T01:14:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39852 deadline: 1733793350330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39834 deadline: 1733793350330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39818 deadline: 1733793350330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:14:50,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/99880bc547e344e7b7072c9246a5a3f8 2024-12-10T01:14:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/63e13defb4a9486b9aa054e12580f600 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600 2024-12-10T01:14:50,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600, entries=200, sequenceid=433, filesize=14.4 K 2024-12-10T01:14:50,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/f7705e8eba9d4af085045d5da6f886aa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa 2024-12-10T01:14:50,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa, entries=150, sequenceid=433, filesize=12.0 K 2024-12-10T01:14:50,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/99880bc547e344e7b7072c9246a5a3f8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8 2024-12-10T01:14:50,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8, entries=150, sequenceid=433, filesize=12.0 K 2024-12-10T01:14:50,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5c85b06919b802618bd08e23219e27af in 1268ms, sequenceid=433, compaction requested=true 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:14:50,419 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:50,419 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c85b06919b802618bd08e23219e27af:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:14:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:50,420 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:50,420 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/A is initiating minor compaction (all files) 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/B is initiating minor compaction (all files) 2024-12-10T01:14:50,421 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/A in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:50,421 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/B in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:50,421 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d7bfb9726b90493b84478ac55155af67, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=51.3 K 2024-12-10T01:14:50,421 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1bcb301cc3af4922b4bcd2698122cefd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=48.9 K 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7bfb9726b90493b84478ac55155af67, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bcb301cc3af4922b4bcd2698122cefd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8b8959627b41f78cca7f1738b5086a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733793284982 2024-12-10T01:14:50,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee4fea4d72841448932ba548ebc023a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733793284982 2024-12-10T01:14:50,422 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f49a39091204baeaf5176bbb45d7e2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733793286334 2024-12-10T01:14:50,422 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5916f752a2304b49bef3bd3c36dd7a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733793286334 2024-12-10T01:14:50,422 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e13defb4a9486b9aa054e12580f600, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733793288541 2024-12-10T01:14:50,422 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f7705e8eba9d4af085045d5da6f886aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733793288541 2024-12-10T01:14:50,430 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#A#compaction#91 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:50,430 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:50,431 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/71029e8e722d4449835deaad4216d860 is 50, key is test_row_0/B:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:50,431 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3cfda4b0323141db84fb4d687e05868e is 50, key is test_row_0/A:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741927_1103 (size=13323) 2024-12-10T01:14:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741928_1104 (size=13323) 2024-12-10T01:14:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T01:14:50,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:50,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T01:14:50,722 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-10T01:14:50,722 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:50,722 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:50,722 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:50,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:50,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/29a7867bf60148cebdc1965aeb86d8b1 is 50, key is test_row_0/A:col10/1733793289188/Put/seqid=0 2024-12-10T01:14:50,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741929_1105 (size=12301) 2024-12-10T01:14:50,847 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/71029e8e722d4449835deaad4216d860 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/71029e8e722d4449835deaad4216d860 2024-12-10T01:14:50,847 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/3cfda4b0323141db84fb4d687e05868e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfda4b0323141db84fb4d687e05868e 2024-12-10T01:14:50,853 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/B of 5c85b06919b802618bd08e23219e27af into 71029e8e722d4449835deaad4216d860(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:50,853 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/A of 5c85b06919b802618bd08e23219e27af into 3cfda4b0323141db84fb4d687e05868e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:50,853 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/B, priority=12, startTime=1733793290419; duration=0sec 2024-12-10T01:14:50,853 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/A, priority=12, startTime=1733793290419; duration=0sec 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:B 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:A 2024-12-10T01:14:50,853 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:14:50,854 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:14:50,854 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5c85b06919b802618bd08e23219e27af/C is initiating minor compaction (all files) 2024-12-10T01:14:50,854 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c85b06919b802618bd08e23219e27af/C in TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:50,854 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/fdad45cce32742efba8f091a4642ccb8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp, totalSize=48.9 K 2024-12-10T01:14:50,855 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting fdad45cce32742efba8f091a4642ccb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733793283814 2024-12-10T01:14:50,855 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ade1eaebb456402e85ba0a9717aad855, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733793284982 2024-12-10T01:14:50,855 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c46dd3283f4d53b467609c2d1c8879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733793286334 2024-12-10T01:14:50,856 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 99880bc547e344e7b7072c9246a5a3f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733793288541 2024-12-10T01:14:50,864 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c85b06919b802618bd08e23219e27af#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:14:50,864 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bd64125785234b509f29e76fe6aa3d44 is 50, key is test_row_0/C:col10/1733793288541/Put/seqid=0 2024-12-10T01:14:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741930_1106 (size=13323) 2024-12-10T01:14:51,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:14:51,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/29a7867bf60148cebdc1965aeb86d8b1 2024-12-10T01:14:51,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c66ba348ef8a4f32ae6d71eced0d07b5 is 50, key is test_row_0/B:col10/1733793289188/Put/seqid=0 2024-12-10T01:14:51,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741931_1107 (size=12301) 2024-12-10T01:14:51,281 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/bd64125785234b509f29e76fe6aa3d44 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bd64125785234b509f29e76fe6aa3d44 2024-12-10T01:14:51,288 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c85b06919b802618bd08e23219e27af/C of 5c85b06919b802618bd08e23219e27af into bd64125785234b509f29e76fe6aa3d44(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:14:51,288 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:51,288 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., storeName=5c85b06919b802618bd08e23219e27af/C, priority=12, startTime=1733793290419; duration=0sec 2024-12-10T01:14:51,288 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:14:51,288 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c85b06919b802618bd08e23219e27af:C 2024-12-10T01:14:51,341 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-10T01:14:51,341 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:51,341 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,341 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:51,351 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-10T01:14:51,351 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-10T01:14:51,351 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:51,351 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:51,351 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,351 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,351 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,351 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:51,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:51,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:51,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c66ba348ef8a4f32ae6d71eced0d07b5 2024-12-10T01:14:51,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da8d223d78dc4637a796919cf973510a is 50, key is test_row_0/C:col10/1733793289188/Put/seqid=0 2024-12-10T01:14:51,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741932_1108 (size=12301) 2024-12-10T01:14:51,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da8d223d78dc4637a796919cf973510a 2024-12-10T01:14:51,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/29a7867bf60148cebdc1965aeb86d8b1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/29a7867bf60148cebdc1965aeb86d8b1 2024-12-10T01:14:51,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/29a7867bf60148cebdc1965aeb86d8b1, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T01:14:51,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/c66ba348ef8a4f32ae6d71eced0d07b5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c66ba348ef8a4f32ae6d71eced0d07b5 2024-12-10T01:14:52,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c66ba348ef8a4f32ae6d71eced0d07b5, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T01:14:52,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/da8d223d78dc4637a796919cf973510a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da8d223d78dc4637a796919cf973510a 2024-12-10T01:14:52,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da8d223d78dc4637a796919cf973510a, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T01:14:52,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=20.13 KB/20610 for 5c85b06919b802618bd08e23219e27af in 1291ms, sequenceid=454, compaction requested=false 2024-12-10T01:14:52,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c85b06919b802618bd08e23219e27af: 2024-12-10T01:14:53,194 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-10T01:14:53,195 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:14:53,195 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:53,195 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:53,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-10T01:14:53,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4031 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3940 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1809 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5422 rows 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1808 2024-12-10T01:14:53,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5420 rows 2024-12-10T01:14:53,197 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:14:53,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:14:53,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:53,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:53,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:14:53,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:14:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:53,210 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793293210"}]},"ts":"1733793293210"} 2024-12-10T01:14:53,211 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:14:53,212 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:14:53,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:14:53,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, UNASSIGN}] 2024-12-10T01:14:53,218 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, UNASSIGN 2024-12-10T01:14:53,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5c85b06919b802618bd08e23219e27af, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:53,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, UNASSIGN because future has completed 2024-12-10T01:14:53,222 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:14:53,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:53,379 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:53,380 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:14:53,380 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 5c85b06919b802618bd08e23219e27af, disabling compactions & flushes 2024-12-10T01:14:53,380 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:53,380 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. after waiting 1 ms 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:53,381 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 5c85b06919b802618bd08e23219e27af 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=A 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=B 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5c85b06919b802618bd08e23219e27af, store=C 2024-12-10T01:14:53,381 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:53,387 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/a35aa3fb22be46b28481bdee6d168148 is 50, key is test_row_0/A:col10/1733793293191/Put/seqid=0 2024-12-10T01:14:53,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741933_1109 (size=12301) 2024-12-10T01:14:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:53,794 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/a35aa3fb22be46b28481bdee6d168148 2024-12-10T01:14:53,805 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/651896c31be3438a97013004170183d8 is 50, key is test_row_0/B:col10/1733793293191/Put/seqid=0 2024-12-10T01:14:53,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741934_1110 (size=12301) 2024-12-10T01:14:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:54,210 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/651896c31be3438a97013004170183d8 2024-12-10T01:14:54,219 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/6eb58df05f2348878e789f3ba518b6f3 is 50, key is test_row_0/C:col10/1733793293191/Put/seqid=0 2024-12-10T01:14:54,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741935_1111 (size=12301) 2024-12-10T01:14:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:54,626 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/6eb58df05f2348878e789f3ba518b6f3 2024-12-10T01:14:54,639 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/A/a35aa3fb22be46b28481bdee6d168148 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/a35aa3fb22be46b28481bdee6d168148 2024-12-10T01:14:54,645 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/a35aa3fb22be46b28481bdee6d168148, entries=150, sequenceid=464, filesize=12.0 K 2024-12-10T01:14:54,646 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/B/651896c31be3438a97013004170183d8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/651896c31be3438a97013004170183d8 2024-12-10T01:14:54,651 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/651896c31be3438a97013004170183d8, entries=150, sequenceid=464, filesize=12.0 K 2024-12-10T01:14:54,652 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/.tmp/C/6eb58df05f2348878e789f3ba518b6f3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/6eb58df05f2348878e789f3ba518b6f3 2024-12-10T01:14:54,658 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/6eb58df05f2348878e789f3ba518b6f3, entries=150, sequenceid=464, filesize=12.0 K 2024-12-10T01:14:54,659 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5c85b06919b802618bd08e23219e27af in 1278ms, sequenceid=464, compaction requested=true 2024-12-10T01:14:54,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/511c431d5bfd469c8d42f2dbbfb190a8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/16e6607f04aa4abb9da92ce4884b375c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ae92434ddd524570a56867fb6331aa9f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/0cc3c1e353af4dfc99d8fa2e550d20c9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/45662c69755747fe900e257fb7c5ea5e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/10d7ac268fc7426988d01780056b5451, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3a2b1a2a12b2463d841e9a82ca6e48fb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d7bfb9726b90493b84478ac55155af67, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600] to archive 2024-12-10T01:14:54,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:14:54,675 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7bff2854907a490d9dd11aff35ee145d 2024-12-10T01:14:54,675 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/511c431d5bfd469c8d42f2dbbfb190a8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/511c431d5bfd469c8d42f2dbbfb190a8 2024-12-10T01:14:54,675 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4dc14f59834440afae0a7e38ac344286 2024-12-10T01:14:54,676 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/4f377a73693442fd96009980f2d8893e 2024-12-10T01:14:54,676 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/16e6607f04aa4abb9da92ce4884b375c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/16e6607f04aa4abb9da92ce4884b375c 2024-12-10T01:14:54,676 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/feecb7a687714e3789ed3a2cf80b4de4 2024-12-10T01:14:54,676 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/7037973b0580413ca801677b58bcae50 2024-12-10T01:14:54,677 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/dc10572f69a34fcdb4bbdb9fb0388fc2 2024-12-10T01:14:54,678 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/bd7b1511dde6462797f1978db7affbe9 2024-12-10T01:14:54,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/07a2cb895e324d7ca89e12a52ba307f9 2024-12-10T01:14:54,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/f869be891aec40b196794da66ae82b13 2024-12-10T01:14:54,679 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfef77fc13045d7a14a615dcb0b94b2 2024-12-10T01:14:54,679 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ae92434ddd524570a56867fb6331aa9f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ae92434ddd524570a56867fb6331aa9f 2024-12-10T01:14:54,680 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/0cc3c1e353af4dfc99d8fa2e550d20c9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/0cc3c1e353af4dfc99d8fa2e550d20c9 2024-12-10T01:14:54,681 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/ddd3a8ce1f5e481c95e5f7c3cebf27ae 2024-12-10T01:14:54,681 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/45662c69755747fe900e257fb7c5ea5e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/45662c69755747fe900e257fb7c5ea5e 2024-12-10T01:14:54,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9191d8d6cfa044b8a747bd433ba7f041 2024-12-10T01:14:54,681 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3e2c873168e24469b85f0070c0a2f7b4 2024-12-10T01:14:54,682 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/98ac2f78601c4e60b31937e9092e9861 2024-12-10T01:14:54,682 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/502a145b61fe4293b2aebdfe18eb8a63 2024-12-10T01:14:54,682 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/10d7ac268fc7426988d01780056b5451 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/10d7ac268fc7426988d01780056b5451 2024-12-10T01:14:54,683 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3a2b1a2a12b2463d841e9a82ca6e48fb to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3a2b1a2a12b2463d841e9a82ca6e48fb 2024-12-10T01:14:54,683 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d4e2ece3c38045e2a22a55c4cdc75803 2024-12-10T01:14:54,683 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e7262da91ee94045ad353d3c64d708f4 2024-12-10T01:14:54,683 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/e5ceb2a8010b485c962acf0399dbfe00 2024-12-10T01:14:54,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/9b8b8959627b41f78cca7f1738b5086a 2024-12-10T01:14:54,684 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/fa34a0fcc93f4b27b5addf8fbe695824 2024-12-10T01:14:54,684 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/8f49a39091204baeaf5176bbb45d7e2a 2024-12-10T01:14:54,684 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/63e13defb4a9486b9aa054e12580f600 2024-12-10T01:14:54,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d7bfb9726b90493b84478ac55155af67 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/d7bfb9726b90493b84478ac55155af67 2024-12-10T01:14:54,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aee16c9799a54fc4a260ab7d0268bd08, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/024e50a35b4f4b9a83e4a124a04e4525, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6cbd965718554f9abd3bc63027e32cc7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6c6e10c2fd694be19cd4021a07d3a895, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/99454f4233804a22988667512e74e830, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/21f95c0094e64886aadcd85d28b84ed2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/4528fbf37d7e409ab7379b4a8978bcf9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1bcb301cc3af4922b4bcd2698122cefd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa] to archive 2024-12-10T01:14:54,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aee16c9799a54fc4a260ab7d0268bd08 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aee16c9799a54fc4a260ab7d0268bd08 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/be8cf60ec4cd46b1a702b4eb258a28db 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c22f611cc0ca4ef992829099d29336d5 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/8d5cdd8327174b8197a535146d793f6f 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/7838763eac134fc8859d2af6bfb7acfc 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/3783f75bd9b94c06ab327b19eb4e1bd6 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/024e50a35b4f4b9a83e4a124a04e4525 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/024e50a35b4f4b9a83e4a124a04e4525 2024-12-10T01:14:54,706 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/aa2c3e0f4f72419e9ac71fb3c9817c36 2024-12-10T01:14:54,708 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6c6e10c2fd694be19cd4021a07d3a895 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6c6e10c2fd694be19cd4021a07d3a895 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1cb24382946f454fa27959e0e9934702 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/ffc1dee74ded4f99b577b5d2787bde28 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c60ac497414c492885dd600e4390f839 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6cbd965718554f9abd3bc63027e32cc7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6cbd965718554f9abd3bc63027e32cc7 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/161a45196ad0446994a1d62e688b1491 2024-12-10T01:14:54,709 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/6834c4d9d5ae4890a75fb32674b71895 2024-12-10T01:14:54,711 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/99454f4233804a22988667512e74e830 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/99454f4233804a22988667512e74e830 2024-12-10T01:14:54,712 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0695176466ba4768959e591f1d4efc31 2024-12-10T01:14:54,712 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5e6eda91485e4a50a5c9baae5b62cff8 2024-12-10T01:14:54,712 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/21f95c0094e64886aadcd85d28b84ed2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/21f95c0094e64886aadcd85d28b84ed2 2024-12-10T01:14:54,712 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c64922d9aad24fd3ad2df9c781df7506 2024-12-10T01:14:54,712 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/4528fbf37d7e409ab7379b4a8978bcf9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/4528fbf37d7e409ab7379b4a8978bcf9 2024-12-10T01:14:54,713 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/e94dc6b34bc648148f998c10b009769c 2024-12-10T01:14:54,713 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/65084c056c8645ee97c9330f3c72b9db 2024-12-10T01:14:54,714 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c12046c32ade4b31a31f93e00d7997d6 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/0ee4fea4d72841448932ba548ebc023a 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/f7705e8eba9d4af085045d5da6f886aa 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/84b7b0a43352453a830f4d1e7926f7ae 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1bcb301cc3af4922b4bcd2698122cefd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/1bcb301cc3af4922b4bcd2698122cefd 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/18c5787c0d3046fd876b0b339c9ed2c8 2024-12-10T01:14:54,716 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/5916f752a2304b49bef3bd3c36dd7a42 2024-12-10T01:14:54,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d1af5094789478dad222e81d1118ea5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/a00c7210e9264fa196fdc5efabbe3c8a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bb05e48e1a6b46deb2189ed099a708d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/721766ba80714d24b47fba8c1f0d3d58, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/1d506d46b24c4a7aa2e309efe1a67c03, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/fdad45cce32742efba8f091a4642ccb8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8] to archive 2024-12-10T01:14:54,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:14:54,720 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f1f7a8b23ffe4f91989473daeeab1090 2024-12-10T01:14:54,720 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da0a89883b4e4fb0a6191deb9f63fa00 2024-12-10T01:14:54,720 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/c5a106275d15481ca5ea057ad6a7d5a3 2024-12-10T01:14:54,720 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddd84712617a41e391a6540a29ada00e 2024-12-10T01:14:54,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/90dd42f4126b48dd8d2c7a0f1fa6d4f0 2024-12-10T01:14:54,721 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ddae1a27f56c409b93b514c7989685be 2024-12-10T01:14:54,721 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/8e5e485b62074a4287d8bbd8c335ac55 2024-12-10T01:14:54,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d1af5094789478dad222e81d1118ea5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d1af5094789478dad222e81d1118ea5 2024-12-10T01:14:54,722 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/9bfd2c8731cf483faa2ee5aa584ceff3 2024-12-10T01:14:54,722 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f409c8cf116a41ebaee94984fc4693fb 2024-12-10T01:14:54,722 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/5d93bbd3c5ed444f952488844b4dd582 2024-12-10T01:14:54,722 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/f95b7640a2fc4419b9aecc401bc7a7f4 2024-12-10T01:14:54,723 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bb05e48e1a6b46deb2189ed099a708d5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bb05e48e1a6b46deb2189ed099a708d5 2024-12-10T01:14:54,723 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/a00c7210e9264fa196fdc5efabbe3c8a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/a00c7210e9264fa196fdc5efabbe3c8a 2024-12-10T01:14:54,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/721766ba80714d24b47fba8c1f0d3d58 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/721766ba80714d24b47fba8c1f0d3d58 2024-12-10T01:14:54,724 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/d351647b18974420919080c49ec3b736 2024-12-10T01:14:54,724 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/061b0448dfd24e49a91d45e895aff16a 2024-12-10T01:14:54,724 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/e6f55786fb4f4f64bcc1ffdf31f5ddc8 2024-12-10T01:14:54,724 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/93e41ca32c64448c8a4a3a6051f0b6dc 2024-12-10T01:14:54,724 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bcc0ed287e82472baefee61039e7f72c 2024-12-10T01:14:54,725 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/959298279d014560b8c918bd90e61b4b 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/1d506d46b24c4a7aa2e309efe1a67c03 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/1d506d46b24c4a7aa2e309efe1a67c03 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/fdad45cce32742efba8f091a4642ccb8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/fdad45cce32742efba8f091a4642ccb8 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/48a84216ce264d69943780b071b3f2c4 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/efb7eeffe5f444bab5b619e3d16a77cf 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/abc5b53b4c5d437e8387552dbf11a9fc 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/102204dbf5744cc2accaa15337e9e372 2024-12-10T01:14:54,726 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/29c46dd3283f4d53b467609c2d1c8879 2024-12-10T01:14:54,727 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/ade1eaebb456402e85ba0a9717aad855 2024-12-10T01:14:54,727 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/99880bc547e344e7b7072c9246a5a3f8 2024-12-10T01:14:54,732 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/recovered.edits/467.seqid, newMaxSeqId=467, maxSeqId=1 2024-12-10T01:14:54,733 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. 2024-12-10T01:14:54,733 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 5c85b06919b802618bd08e23219e27af: Waiting for close lock at 1733793293380Running coprocessor pre-close hooks at 1733793293380Disabling compacts and flushes for region at 1733793293380Disabling writes for close at 1733793293381 (+1 ms)Obtaining lock to block concurrent updates at 1733793293381Preparing flush snapshotting stores in 5c85b06919b802618bd08e23219e27af at 1733793293381Finished memstore snapshotting TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733793293381Flushing stores of TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af. at 1733793293382 (+1 ms)Flushing 5c85b06919b802618bd08e23219e27af/A: creating writer at 1733793293382Flushing 5c85b06919b802618bd08e23219e27af/A: appending metadata at 1733793293387 (+5 ms)Flushing 5c85b06919b802618bd08e23219e27af/A: closing flushed file at 1733793293387Flushing 5c85b06919b802618bd08e23219e27af/B: creating writer at 1733793293801 (+414 ms)Flushing 5c85b06919b802618bd08e23219e27af/B: appending metadata at 1733793293804 (+3 ms)Flushing 5c85b06919b802618bd08e23219e27af/B: closing flushed file at 1733793293804Flushing 5c85b06919b802618bd08e23219e27af/C: creating writer at 1733793294216 (+412 ms)Flushing 5c85b06919b802618bd08e23219e27af/C: appending metadata at 1733793294219 (+3 ms)Flushing 5c85b06919b802618bd08e23219e27af/C: closing flushed file at 1733793294219Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e9cb3e: reopening flushed file at 1733793294638 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@608a8ca7: reopening flushed file at 1733793294645 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41fe6163: reopening flushed file at 1733793294651 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5c85b06919b802618bd08e23219e27af in 1278ms, sequenceid=464, compaction requested=true at 1733793294659 (+8 ms)Writing region close event to WAL at 1733793294728 (+69 ms)Running coprocessor post-close hooks at 1733793294732 (+4 ms)Closed at 1733793294732 2024-12-10T01:14:54,735 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:54,735 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5c85b06919b802618bd08e23219e27af, regionState=CLOSED 2024-12-10T01:14:54,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:14:54,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-10T01:14:54,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 5c85b06919b802618bd08e23219e27af, server=fd1a2b66caf8,38925,1733793262612 in 1.5160 sec 2024-12-10T01:14:54,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-10T01:14:54,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c85b06919b802618bd08e23219e27af, UNASSIGN in 1.5230 sec 2024-12-10T01:14:54,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-10T01:14:54,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5290 sec 2024-12-10T01:14:54,745 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793294745"}]},"ts":"1733793294745"} 2024-12-10T01:14:54,746 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:14:54,746 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:14:54,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5420 sec 2024-12-10T01:14:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T01:14:55,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:55,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:14:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,384 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T01:14:55,386 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,390 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:55,395 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/recovered.edits] 2024-12-10T01:14:55,398 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/29a7867bf60148cebdc1965aeb86d8b1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/29a7867bf60148cebdc1965aeb86d8b1 2024-12-10T01:14:55,398 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/a35aa3fb22be46b28481bdee6d168148 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/a35aa3fb22be46b28481bdee6d168148 2024-12-10T01:14:55,398 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfda4b0323141db84fb4d687e05868e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/A/3cfda4b0323141db84fb4d687e05868e 2024-12-10T01:14:55,401 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/651896c31be3438a97013004170183d8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/651896c31be3438a97013004170183d8 2024-12-10T01:14:55,401 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/71029e8e722d4449835deaad4216d860 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/71029e8e722d4449835deaad4216d860 2024-12-10T01:14:55,401 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c66ba348ef8a4f32ae6d71eced0d07b5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/B/c66ba348ef8a4f32ae6d71eced0d07b5 2024-12-10T01:14:55,404 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/6eb58df05f2348878e789f3ba518b6f3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/6eb58df05f2348878e789f3ba518b6f3 2024-12-10T01:14:55,404 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bd64125785234b509f29e76fe6aa3d44 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/bd64125785234b509f29e76fe6aa3d44 2024-12-10T01:14:55,404 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da8d223d78dc4637a796919cf973510a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/C/da8d223d78dc4637a796919cf973510a 2024-12-10T01:14:55,407 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/recovered.edits/467.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af/recovered.edits/467.seqid 2024-12-10T01:14:55,407 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5c85b06919b802618bd08e23219e27af 2024-12-10T01:14:55,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:14:55,412 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T01:14:55,425 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:14:55,463 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:14:55,465 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,465 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:14:55,465 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793295465"}]},"ts":"9223372036854775807"} 2024-12-10T01:14:55,468 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:14:55,468 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5c85b06919b802618bd08e23219e27af, NAME => 'TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:14:55,469 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:14:55,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793295469"}]},"ts":"9223372036854775807"} 2024-12-10T01:14:55,471 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:14:55,472 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-12-10T01:14:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T01:14:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:14:55,490 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:55,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=255 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1274070357_22 at /127.0.0.1:36158 [Waiting for operation #295] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fd1a2b66caf8:38925-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_849766755_22 at /127.0.0.1:33442 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 420) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7328 (was 7953) 2024-12-10T01:14:55,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=7328 2024-12-10T01:14:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:14:55,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:14:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:55,517 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:14:55,517 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:55,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-12-10T01:14:55,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:14:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:55,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741936_1112 (size=1061) 2024-12-10T01:14:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:55,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:14:55,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741937_1113 (size=53) 2024-12-10T01:14:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:56,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:56,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5252bc35c0b22e4dae6eb027696caf9a, disabling compactions & flushes 2024-12-10T01:14:56,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. after waiting 0 ms 2024-12-10T01:14:56,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5252bc35c0b22e4dae6eb027696caf9a: Waiting for close lock at 1733793296342Disabling compacts and flushes for region at 1733793296342Disabling writes for close at 1733793296343 (+1 ms)Writing region close event to WAL at 1733793296343Closed at 1733793296343 2024-12-10T01:14:56,345 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:14:56,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793296345"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793296345"}]},"ts":"1733793296345"} 2024-12-10T01:14:56,349 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:14:56,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:14:56,350 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793296350"}]},"ts":"1733793296350"} 2024-12-10T01:14:56,351 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:14:56,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, ASSIGN}] 2024-12-10T01:14:56,352 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, ASSIGN 2024-12-10T01:14:56,353 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:14:56,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:56,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, ASSIGN because future has completed 2024-12-10T01:14:56,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:56,671 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,671 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:14:56,672 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,673 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:56,673 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,673 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,677 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,678 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:56,679 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName A 2024-12-10T01:14:56,679 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:56,679 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:56,679 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:56,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName B 2024-12-10T01:14:56,681 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:56,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:56,682 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,683 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:56,683 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName C 2024-12-10T01:14:56,683 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:56,683 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:56,684 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,684 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,684 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,685 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,686 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,686 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,686 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:14:56,688 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,690 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:14:56,690 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 5252bc35c0b22e4dae6eb027696caf9a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73243357, jitterRate=0.09141106903553009}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:14:56,690 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:56,692 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 5252bc35c0b22e4dae6eb027696caf9a: Running coprocessor pre-open hook at 1733793296674Writing region info on filesystem at 1733793296674Initializing all the Stores at 1733793296676 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793296676Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793296677 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793296677Cleaning up temporary data from old regions at 1733793296686 (+9 ms)Running coprocessor post-open hooks at 1733793296690 (+4 ms)Region opened successfully at 1733793296692 (+2 ms) 2024-12-10T01:14:56,693 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., pid=26, masterSystemTime=1733793296664 2024-12-10T01:14:56,695 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,696 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:56,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:56,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:14:56,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-10T01:14:56,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 in 189 msec 2024-12-10T01:14:56,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-10T01:14:56,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, ASSIGN in 351 msec 2024-12-10T01:14:56,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:14:56,706 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793296705"}]},"ts":"1733793296705"} 2024-12-10T01:14:56,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:14:56,708 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:14:56,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-12-10T01:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-10T01:14:57,670 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:57,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aed7837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:57,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:57,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:57,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336396c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:57,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f69af1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:57,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:57,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:57,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1add8201 2024-12-10T01:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:14:57,687 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:14:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:14:57,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:14:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T01:14:57,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741938_1114 (size=1097) 2024-12-10T01:14:58,113 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T01:14:58,113 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T01:14:58,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:14:58,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE}] 2024-12-10T01:14:58,124 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE 2024-12-10T01:14:58,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE because future has completed 2024-12-10T01:14:58,127 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:14:58,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:58,280 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,280 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:14:58,281 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 5252bc35c0b22e4dae6eb027696caf9a, disabling compactions & flushes 2024-12-10T01:14:58,281 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,281 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,281 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. after waiting 0 ms 2024-12-10T01:14:58,281 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,339 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T01:14:58,341 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,342 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 5252bc35c0b22e4dae6eb027696caf9a: Waiting for close lock at 1733793298281Running coprocessor pre-close hooks at 1733793298281Disabling compacts and flushes for region at 1733793298281Disabling writes for close at 1733793298281Writing region close event to WAL at 1733793298282 (+1 ms)Running coprocessor post-close hooks at 1733793298341 (+59 ms)Closed at 1733793298341 2024-12-10T01:14:58,342 WARN [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 5252bc35c0b22e4dae6eb027696caf9a to self. 2024-12-10T01:14:58,347 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=CLOSED 2024-12-10T01:14:58,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:14:58,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-10T01:14:58,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 in 224 msec 2024-12-10T01:14:58,354 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE; state=CLOSED, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=true 2024-12-10T01:14:58,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE because future has completed 2024-12-10T01:14:58,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:14:58,665 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,665 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:14:58,666 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,666 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:14:58,666 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,666 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,671 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,672 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:58,677 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName A 2024-12-10T01:14:58,679 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:58,680 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:58,680 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:58,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName B 2024-12-10T01:14:58,681 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:58,681 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:58,682 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,682 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:14:58,682 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5252bc35c0b22e4dae6eb027696caf9a columnFamilyName C 2024-12-10T01:14:58,682 DEBUG [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:58,683 INFO [StoreOpener-5252bc35c0b22e4dae6eb027696caf9a-1 {}] regionserver.HStore(327): Store=5252bc35c0b22e4dae6eb027696caf9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:14:58,683 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,683 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,684 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,685 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,685 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,685 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,686 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:14:58,687 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,687 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 5252bc35c0b22e4dae6eb027696caf9a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60936849, jitterRate=-0.09197019040584564}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:14:58,688 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,688 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 5252bc35c0b22e4dae6eb027696caf9a: Running coprocessor pre-open hook at 1733793298666Writing region info on filesystem at 1733793298666Initializing all the Stores at 1733793298668 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793298668Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793298668Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793298668Cleaning up temporary data from old regions at 1733793298685 (+17 ms)Running coprocessor post-open hooks at 1733793298688 (+3 ms)Region opened successfully at 1733793298688 2024-12-10T01:14:58,689 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., pid=31, masterSystemTime=1733793298658 2024-12-10T01:14:58,691 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,691 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,691 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=OPEN, openSeqNum=5, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:14:58,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-12-10T01:14:58,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 in 188 msec 2024-12-10T01:14:58,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-10T01:14:58,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, REOPEN/MOVE in 572 msec 2024-12-10T01:14:58,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-10T01:14:58,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 583 msec 2024-12-10T01:14:58,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0070 sec 2024-12-10T01:14:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T01:14:58,707 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:14:58,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0f2208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3768576f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c5f290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2696e985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,714 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696adb49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,715 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb8153a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef31d00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1a735d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9a9115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ccc7654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa59292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b51daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239184b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100177ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727a92ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3bf539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,731 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562b335d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6663af79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4d6873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,734 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754865c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f2b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67462735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bda3ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,739 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399b9222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17419201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:14:58,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:14:58,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:14:58,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:14:58,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:14:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9bc57a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:14:58,745 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,745 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,745 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,747 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,747 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,753 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:14:58,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,758 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-12-10T01:14:58,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,760 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:14:58,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:14:58,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,761 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:14:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:14:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:14:58,763 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:14:58,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:14:58,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:14:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:58,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:14:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:58,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c78bf0293864719af98e0a055121b8a_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_1/A:col10/1733793298769/Put/seqid=0 2024-12-10T01:14:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793358823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793358823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793298798, max timestamp=1733793298826Num uniques 859; 2024-12-10T01:14:58,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:58,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793298798, max timestamp=1733793298826Num uniques 859; 2024-12-10T01:14:58,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:58,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793298798, max timestamp=1733793298826Num uniques 859; 2024-12-10T01:14:58,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741939_1115 (size=14594) 2024-12-10T01:14:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:14:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793358876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793358878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793358878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:58,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:58,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793358940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793358940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793358990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793358999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793358999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:14:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793359149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793359149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793359200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793359210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793359210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:59,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c78bf0293864719af98e0a055121b8a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c78bf0293864719af98e0a055121b8a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:59,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b9f0aedb91d945e4b7f0cc4c009e26c5, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:14:59,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b9f0aedb91d945e4b7f0cc4c009e26c5 is 175, key is test_row_1/A:col10/1733793298769/Put/seqid=0 2024-12-10T01:14:59,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741940_1116 (size=39545) 2024-12-10T01:14:59,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b9f0aedb91d945e4b7f0cc4c009e26c5 2024-12-10T01:14:59,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/ae63abac07134511b9c8a13f3da06ec1 is 50, key is test_row_1/B:col10/1733793298769/Put/seqid=0 2024-12-10T01:14:59,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741941_1117 (size=9657) 2024-12-10T01:14:59,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/ae63abac07134511b9c8a13f3da06ec1 2024-12-10T01:14:59,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4d0452b8586742b99a9df72fe07083fa is 50, key is test_row_1/C:col10/1733793298769/Put/seqid=0 2024-12-10T01:14:59,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741942_1118 (size=9657) 2024-12-10T01:14:59,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4d0452b8586742b99a9df72fe07083fa 2024-12-10T01:14:59,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b9f0aedb91d945e4b7f0cc4c009e26c5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5 2024-12-10T01:14:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:14:59,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5, entries=200, sequenceid=15, filesize=38.6 K 2024-12-10T01:14:59,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/ae63abac07134511b9c8a13f3da06ec1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1 2024-12-10T01:14:59,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1, entries=100, sequenceid=15, filesize=9.4 K 2024-12-10T01:14:59,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4d0452b8586742b99a9df72fe07083fa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa 2024-12-10T01:14:59,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa, entries=100, sequenceid=15, filesize=9.4 K 2024-12-10T01:14:59,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5252bc35c0b22e4dae6eb027696caf9a in 649ms, sequenceid=15, compaction requested=false 2024-12-10T01:14:59,419 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T01:14:59,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:14:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:59,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T01:14:59,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:14:59,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:59,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:14:59,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:59,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:14:59,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:14:59,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fb3c4277413d4d228c2c9c8ea1fde210_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793298876/Put/seqid=0 2024-12-10T01:14:59,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741943_1119 (size=14594) 2024-12-10T01:14:59,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:59,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fb3c4277413d4d228c2c9c8ea1fde210_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb3c4277413d4d228c2c9c8ea1fde210_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:59,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b5fde365447d4c968132e4c474f63235, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:14:59,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b5fde365447d4c968132e4c474f63235 is 175, key is test_row_0/A:col10/1733793298876/Put/seqid=0 2024-12-10T01:14:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793359506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793359506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741944_1120 (size=39549) 2024-12-10T01:14:59,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b5fde365447d4c968132e4c474f63235 2024-12-10T01:14:59,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793359520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793359520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793359520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/acadfebbffb44da5ad5f414b55b118df is 50, key is test_row_0/B:col10/1733793298876/Put/seqid=0 2024-12-10T01:14:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741945_1121 (size=12001) 2024-12-10T01:14:59,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/acadfebbffb44da5ad5f414b55b118df 2024-12-10T01:14:59,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/32811c65e8354a1798c1dc0d67f15e58 is 50, key is test_row_0/C:col10/1733793298876/Put/seqid=0 2024-12-10T01:14:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741946_1122 (size=12001) 2024-12-10T01:14:59,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/32811c65e8354a1798c1dc0d67f15e58 2024-12-10T01:14:59,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b5fde365447d4c968132e4c474f63235 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235 2024-12-10T01:14:59,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235, entries=200, sequenceid=52, filesize=38.6 K 2024-12-10T01:14:59,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/acadfebbffb44da5ad5f414b55b118df as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df 2024-12-10T01:14:59,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T01:14:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/32811c65e8354a1798c1dc0d67f15e58 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58 2024-12-10T01:14:59,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T01:14:59,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5252bc35c0b22e4dae6eb027696caf9a in 156ms, sequenceid=52, compaction requested=false 2024-12-10T01:14:59,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:14:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:59,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:14:59,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101f0ab4fce1d74d7aa6218ce3c786db79_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793299489/Put/seqid=0 2024-12-10T01:14:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741947_1123 (size=12154) 2024-12-10T01:14:59,675 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:14:59,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101f0ab4fce1d74d7aa6218ce3c786db79_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101f0ab4fce1d74d7aa6218ce3c786db79_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:14:59,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/efc80766eead4a2597b329014120b355, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:14:59,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/efc80766eead4a2597b329014120b355 is 175, key is test_row_0/A:col10/1733793299489/Put/seqid=0 2024-12-10T01:14:59,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741948_1124 (size=30955) 2024-12-10T01:14:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/efc80766eead4a2597b329014120b355 2024-12-10T01:14:59,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/22bc3d8391de49738edfb711d396633a is 50, key is test_row_0/B:col10/1733793299489/Put/seqid=0 2024-12-10T01:14:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741949_1125 (size=12001) 2024-12-10T01:14:59,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793299664, max timestamp=1733793299739Num uniques 859; 2024-12-10T01:14:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793299664, max timestamp=1733793299739Num uniques 859; 2024-12-10T01:14:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:14:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793299664, max timestamp=1733793299739Num uniques 859; 2024-12-10T01:14:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:14:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:14:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793359796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793359804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:14:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793359910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:14:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793359919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:14:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:14:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:14:59,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:14:59,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793360039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793360039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793360039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793360119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/22bc3d8391de49738edfb711d396633a 2024-12-10T01:15:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793360129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/bd8bd9477b7d415b8f8c03aadaa5261e is 50, key is test_row_0/C:col10/1733793299489/Put/seqid=0 2024-12-10T01:15:00,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741950_1126 (size=12001) 2024-12-10T01:15:00,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:15:00,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:00,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:15:00,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:15:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793360430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793360439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:00,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:15:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:00,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/bd8bd9477b7d415b8f8c03aadaa5261e 2024-12-10T01:15:00,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/efc80766eead4a2597b329014120b355 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355 2024-12-10T01:15:00,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355, entries=150, sequenceid=64, filesize=30.2 K 2024-12-10T01:15:00,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/22bc3d8391de49738edfb711d396633a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a 2024-12-10T01:15:00,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a, entries=150, sequenceid=64, filesize=11.7 K 2024-12-10T01:15:00,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/bd8bd9477b7d415b8f8c03aadaa5261e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e 2024-12-10T01:15:00,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e, entries=150, sequenceid=64, filesize=11.7 K 2024-12-10T01:15:00,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 5252bc35c0b22e4dae6eb027696caf9a in 976ms, sequenceid=64, compaction requested=true 2024-12-10T01:15:00,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:00,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:00,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:00,608 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:00,609 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:00,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T01:15:00,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,613 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-10T01:15:00,613 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:00,613 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/A is initiating minor compaction (all files) 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:00,613 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/A in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:00,613 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=107.5 K 2024-12-10T01:15:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:00,613 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355] 2024-12-10T01:15:00,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:00,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/B is initiating minor compaction (all files) 2024-12-10T01:15:00,615 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/B in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,616 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=32.9 K 2024-12-10T01:15:00,616 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9f0aedb91d945e4b7f0cc4c009e26c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793298762 2024-12-10T01:15:00,616 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ae63abac07134511b9c8a13f3da06ec1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793298767 2024-12-10T01:15:00,617 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5fde365447d4c968132e4c474f63235, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793298849 2024-12-10T01:15:00,617 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting acadfebbffb44da5ad5f414b55b118df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793298871 2024-12-10T01:15:00,617 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc80766eead4a2597b329014120b355, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:00,617 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 22bc3d8391de49738edfb711d396633a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:00,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a5d51253e3764fa9ad6758df93de3abe_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793299802/Put/seqid=0 2024-12-10T01:15:00,654 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:00,654 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/0fc604463aa4440ea781db898c65dac4 is 50, key is test_row_0/B:col10/1733793299489/Put/seqid=0 2024-12-10T01:15:00,695 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:00,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741951_1127 (size=12154) 2024-12-10T01:15:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,729 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210458e380b56bb4b39a74b06c11bcc4aef_5252bc35c0b22e4dae6eb027696caf9a store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:00,731 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a5d51253e3764fa9ad6758df93de3abe_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a5d51253e3764fa9ad6758df93de3abe_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:00,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/be26520cb80246499daaf6648d91dc6a, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:00,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/be26520cb80246499daaf6648d91dc6a is 175, key is test_row_0/A:col10/1733793299802/Put/seqid=0 2024-12-10T01:15:00,739 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210458e380b56bb4b39a74b06c11bcc4aef_5252bc35c0b22e4dae6eb027696caf9a, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:00,740 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210458e380b56bb4b39a74b06c11bcc4aef_5252bc35c0b22e4dae6eb027696caf9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741952_1128 (size=12104) 2024-12-10T01:15:00,780 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/0fc604463aa4440ea781db898c65dac4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0fc604463aa4440ea781db898c65dac4 2024-12-10T01:15:00,788 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/B of 5252bc35c0b22e4dae6eb027696caf9a into 0fc604463aa4440ea781db898c65dac4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:00,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:00,788 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/B, priority=13, startTime=1733793300609; duration=0sec 2024-12-10T01:15:00,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:00,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:B 2024-12-10T01:15:00,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:00,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:00,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/C is initiating minor compaction (all files) 2024-12-10T01:15:00,790 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/C in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,790 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=32.9 K 2024-12-10T01:15:00,791 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0452b8586742b99a9df72fe07083fa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793298767 2024-12-10T01:15:00,792 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 32811c65e8354a1798c1dc0d67f15e58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793298871 2024-12-10T01:15:00,793 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8bd9477b7d415b8f8c03aadaa5261e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741953_1129 (size=30955) 2024-12-10T01:15:00,802 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/be26520cb80246499daaf6648d91dc6a 2024-12-10T01:15:00,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741954_1130 (size=4469) 2024-12-10T01:15:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/a479a58dfe3244369a761db6450d65a7 is 50, key is test_row_0/B:col10/1733793299802/Put/seqid=0 2024-12-10T01:15:00,847 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#C#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:00,848 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/74b6ed257dd2449a9b8311cd566a10ab is 50, key is test_row_0/C:col10/1733793299489/Put/seqid=0 2024-12-10T01:15:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741955_1131 (size=12001) 2024-12-10T01:15:00,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/a479a58dfe3244369a761db6450d65a7 2024-12-10T01:15:00,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/15d40c1ce3be4b809d9a6ed6af14cc55 is 50, key is test_row_0/C:col10/1733793299802/Put/seqid=0 2024-12-10T01:15:00,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741956_1132 (size=12104) 2024-12-10T01:15:00,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741957_1133 (size=12001) 2024-12-10T01:15:00,909 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/74b6ed257dd2449a9b8311cd566a10ab as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/74b6ed257dd2449a9b8311cd566a10ab 2024-12-10T01:15:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:15:00,912 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/15d40c1ce3be4b809d9a6ed6af14cc55 2024-12-10T01:15:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/be26520cb80246499daaf6648d91dc6a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a 2024-12-10T01:15:00,925 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/C of 5252bc35c0b22e4dae6eb027696caf9a into 74b6ed257dd2449a9b8311cd566a10ab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:00,925 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:00,925 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/C, priority=13, startTime=1733793300609; duration=0sec 2024-12-10T01:15:00,925 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:00,925 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:C 2024-12-10T01:15:00,928 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a, entries=150, sequenceid=98, filesize=30.2 K 2024-12-10T01:15:00,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/a479a58dfe3244369a761db6450d65a7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7 2024-12-10T01:15:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,937 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:15:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/15d40c1ce3be4b809d9a6ed6af14cc55 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55 2024-12-10T01:15:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:15:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,956 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for 5252bc35c0b22e4dae6eb027696caf9a in 343ms, sequenceid=98, compaction requested=false 2024-12-10T01:15:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-10T01:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-10T01:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-10T01:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-12-10T01:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 2.2050 sec 2024-12-10T01:15:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:01,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:01,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:01,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:01,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:01,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:01,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:01,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210db5bfbb949fd48b0b0f67bd3cc27f305_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741958_1134 (size=21918) 2024-12-10T01:15:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733793301058, max timestamp=1733793301101Num uniques 859; 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733793301058, max timestamp=1733793301101Num uniques 859; 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733793301058, max timestamp=1733793301101Num uniques 859; 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T01:15:01,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T01:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793361104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793361111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793361112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793361117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793361117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793361220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793361220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793361220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,225 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#A#compaction#110 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,226 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/856b99e687714c9c89a535652412dbbf is 175, key is test_row_0/A:col10/1733793299489/Put/seqid=0 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741959_1135 (size=31058) 2024-12-10T01:15:01,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793361229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793361229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,246 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/856b99e687714c9c89a535652412dbbf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf 2024-12-10T01:15:01,258 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/A of 5252bc35c0b22e4dae6eb027696caf9a into 856b99e687714c9c89a535652412dbbf(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:01,258 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:01,258 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/A, priority=13, startTime=1733793300608; duration=0sec 2024-12-10T01:15:01,258 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:01,258 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:A 2024-12-10T01:15:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793361430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793361430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793361430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793361449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793361449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:01,516 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210db5bfbb949fd48b0b0f67bd3cc27f305_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db5bfbb949fd48b0b0f67bd3cc27f305_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:01,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cb039948945241e6a032ccee7ac0983d, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:01,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cb039948945241e6a032ccee7ac0983d is 175, key is test_row_0/A:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741960_1136 (size=65323) 2024-12-10T01:15:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793361740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793361749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793361749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793361769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793361769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:01,946 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cb039948945241e6a032ccee7ac0983d 2024-12-10T01:15:01,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/79a04b733aa340af9613b62192ac31a3 is 50, key is test_row_0/B:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741961_1137 (size=12001) 2024-12-10T01:15:02,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793362249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:02,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793362259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793362259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:02,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793362280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:02,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793362280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:02,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/79a04b733aa340af9613b62192ac31a3 2024-12-10T01:15:02,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/7417640dc7024cd0bc776dc92362ab50 is 50, key is test_row_0/C:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:02,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741962_1138 (size=12001) 2024-12-10T01:15:02,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/7417640dc7024cd0bc776dc92362ab50 2024-12-10T01:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cb039948945241e6a032ccee7ac0983d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d 2024-12-10T01:15:02,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d, entries=350, sequenceid=110, filesize=63.8 K 2024-12-10T01:15:02,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/79a04b733aa340af9613b62192ac31a3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3 2024-12-10T01:15:02,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3, entries=150, sequenceid=110, filesize=11.7 K 2024-12-10T01:15:02,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/7417640dc7024cd0bc776dc92362ab50 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50 2024-12-10T01:15:02,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50, entries=150, sequenceid=110, filesize=11.7 K 2024-12-10T01:15:02,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 5252bc35c0b22e4dae6eb027696caf9a in 1350ms, sequenceid=110, compaction requested=true 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:02,405 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:02,405 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:02,406 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:02,406 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/A is initiating minor compaction (all files) 2024-12-10T01:15:02,406 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/A in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:02,406 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=124.4 K 2024-12-10T01:15:02,407 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:02,407 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d] 2024-12-10T01:15:02,407 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:02,407 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/B is initiating minor compaction (all files) 2024-12-10T01:15:02,407 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/B in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:02,407 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0fc604463aa4440ea781db898c65dac4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=35.3 K 2024-12-10T01:15:02,407 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856b99e687714c9c89a535652412dbbf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:02,408 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc604463aa4440ea781db898c65dac4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:02,408 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting be26520cb80246499daaf6648d91dc6a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793299784 2024-12-10T01:15:02,408 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a479a58dfe3244369a761db6450d65a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793299784 2024-12-10T01:15:02,408 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb039948945241e6a032ccee7ac0983d, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301000 2024-12-10T01:15:02,409 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a04b733aa340af9613b62192ac31a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301043 2024-12-10T01:15:02,418 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:02,419 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/7af805d433dc493b94d816a9949a44a9 is 50, key is test_row_0/B:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:02,420 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:02,422 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210082aeb8e9b3e46cc83835b26299fdf12_5252bc35c0b22e4dae6eb027696caf9a store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:02,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741963_1139 (size=12207) 2024-12-10T01:15:02,424 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210082aeb8e9b3e46cc83835b26299fdf12_5252bc35c0b22e4dae6eb027696caf9a, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:02,424 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210082aeb8e9b3e46cc83835b26299fdf12_5252bc35c0b22e4dae6eb027696caf9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741964_1140 (size=4469) 2024-12-10T01:15:02,829 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/7af805d433dc493b94d816a9949a44a9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7af805d433dc493b94d816a9949a44a9 2024-12-10T01:15:02,830 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#A#compaction#118 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:02,830 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ccb55cc8dd74492285027772b10d5a13 is 175, key is test_row_0/A:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:02,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741965_1141 (size=31161) 2024-12-10T01:15:02,836 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/B of 5252bc35c0b22e4dae6eb027696caf9a into 7af805d433dc493b94d816a9949a44a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:02,836 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:02,836 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/B, priority=13, startTime=1733793302405; duration=0sec 2024-12-10T01:15:02,837 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:02,837 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:B 2024-12-10T01:15:02,837 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:02,838 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:02,838 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/C is initiating minor compaction (all files) 2024-12-10T01:15:02,838 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/C in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:02,838 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/74b6ed257dd2449a9b8311cd566a10ab, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=35.3 K 2024-12-10T01:15:02,838 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b6ed257dd2449a9b8311cd566a10ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733793299489 2024-12-10T01:15:02,839 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d40c1ce3be4b809d9a6ed6af14cc55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793299784 2024-12-10T01:15:02,839 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7417640dc7024cd0bc776dc92362ab50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301043 2024-12-10T01:15:02,849 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:02,849 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/0f4dc32bf80841c2b921547f95e02c0a is 50, key is test_row_0/C:col10/1733793301043/Put/seqid=0 2024-12-10T01:15:02,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741966_1142 (size=12207) 2024-12-10T01:15:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T01:15:02,929 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:02,930 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-10T01:15:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-10T01:15:02,933 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:02,933 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:02,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-10T01:15:03,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-10T01:15:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-12-10T01:15:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b5a241ac644c4d05b960b310cf74dc8a_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793301108/Put/seqid=0 2024-12-10T01:15:03,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741967_1143 (size=12304) 2024-12-10T01:15:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,109 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b5a241ac644c4d05b960b310cf74dc8a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5a241ac644c4d05b960b310cf74dc8a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:03,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b2206e0cfacf4253967c8757edb2cb14, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b2206e0cfacf4253967c8757edb2cb14 is 175, key is test_row_0/A:col10/1733793301108/Put/seqid=0 2024-12-10T01:15:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741968_1144 (size=31105) 2024-12-10T01:15:03,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b2206e0cfacf4253967c8757edb2cb14 2024-12-10T01:15:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/fd044c8cc1164bd0947b63e39dd84320 is 50, key is test_row_0/B:col10/1733793301108/Put/seqid=0 2024-12-10T01:15:03,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741969_1145 (size=12151) 2024-12-10T01:15:03,138 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/fd044c8cc1164bd0947b63e39dd84320 2024-12-10T01:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/3de7590b7c964d678a8636c8a4e8b47a is 50, key is test_row_0/C:col10/1733793301108/Put/seqid=0 2024-12-10T01:15:03,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741970_1146 (size=12151) 2024-12-10T01:15:03,168 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/3de7590b7c964d678a8636c8a4e8b47a 2024-12-10T01:15:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/b2206e0cfacf4253967c8757edb2cb14 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14 2024-12-10T01:15:03,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14, entries=150, sequenceid=138, filesize=30.4 K 2024-12-10T01:15:03,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/fd044c8cc1164bd0947b63e39dd84320 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320 2024-12-10T01:15:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320, entries=150, sequenceid=138, filesize=11.9 K 2024-12-10T01:15:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/3de7590b7c964d678a8636c8a4e8b47a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a 2024-12-10T01:15:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a, entries=150, sequenceid=138, filesize=11.9 K 2024-12-10T01:15:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=0 B/0 for 5252bc35c0b22e4dae6eb027696caf9a in 114ms, sequenceid=138, compaction requested=false 2024-12-10T01:15:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-10T01:15:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-10T01:15:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-10T01:15:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 269 msec 2024-12-10T01:15:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 276 msec 2024-12-10T01:15:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,247 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ccb55cc8dd74492285027772b10d5a13 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13 2024-12-10T01:15:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-10T01:15:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,250 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-10T01:15:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:03,255 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/A of 5252bc35c0b22e4dae6eb027696caf9a into ccb55cc8dd74492285027772b10d5a13(size=30.4 K), total size for store is 60.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:03,255 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,255 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/A, priority=13, startTime=1733793302405; duration=0sec 2024-12-10T01:15:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:03,255 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:03,255 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:A 2024-12-10T01:15:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/0f4dc32bf80841c2b921547f95e02c0a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0f4dc32bf80841c2b921547f95e02c0a 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,276 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/C of 5252bc35c0b22e4dae6eb027696caf9a into 0f4dc32bf80841c2b921547f95e02c0a(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:03,276 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,276 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/C, priority=13, startTime=1733793302405; duration=0sec 2024-12-10T01:15:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,276 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:03,276 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:C 2024-12-10T01:15:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:03,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108cbbf2a8db004683a3facdc9f3c9c0ee_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741971_1147 (size=24758) 2024-12-10T01:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:03,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108cbbf2a8db004683a3facdc9f3c9c0ee_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108cbbf2a8db004683a3facdc9f3c9c0ee_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:03,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/a56cbc12f8c94b20a323ce65fdd6c7fc, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/a56cbc12f8c94b20a323ce65fdd6c7fc is 175, key is test_row_0/A:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741972_1148 (size=74395) 2024-12-10T01:15:03,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/a56cbc12f8c94b20a323ce65fdd6c7fc 2024-12-10T01:15:03,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793363391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793303375, max timestamp=1733793303402Num uniques 859; 2024-12-10T01:15:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:15:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793303375, max timestamp=1733793303402Num uniques 859; 2024-12-10T01:15:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793363390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793303375, max timestamp=1733793303402Num uniques 859; 2024-12-10T01:15:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:15:03,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:03,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c7466c009c2549c7b3136592be297159 is 50, key is test_row_0/B:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741973_1149 (size=12151) 2024-12-10T01:15:03,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c7466c009c2549c7b3136592be297159 2024-12-10T01:15:03,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/d4aafa342a2249ccaf8087277f1a991d is 50, key is test_row_0/C:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793363434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793363436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793363443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741974_1150 (size=12151) 2024-12-10T01:15:03,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/d4aafa342a2249ccaf8087277f1a991d 2024-12-10T01:15:03,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/a56cbc12f8c94b20a323ce65fdd6c7fc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc 2024-12-10T01:15:03,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc, entries=400, sequenceid=151, filesize=72.7 K 2024-12-10T01:15:03,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c7466c009c2549c7b3136592be297159 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159 2024-12-10T01:15:03,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T01:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/d4aafa342a2249ccaf8087277f1a991d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d 2024-12-10T01:15:03,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T01:15:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793363510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5252bc35c0b22e4dae6eb027696caf9a in 165ms, sequenceid=151, compaction requested=true 2024-12-10T01:15:03,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:03,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:03,516 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:03,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:03,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:03,516 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:03,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:03,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:03,517 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:03,517 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:03,517 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/B is initiating minor compaction (all files) 2024-12-10T01:15:03,517 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/B in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,518 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7af805d433dc493b94d816a9949a44a9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=35.7 K 2024-12-10T01:15:03,518 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/A is initiating minor compaction (all files) 2024-12-10T01:15:03,518 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/A in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,518 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=133.5 K 2024-12-10T01:15:03,518 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,518 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc] 2024-12-10T01:15:03,519 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af805d433dc493b94d816a9949a44a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301043 2024-12-10T01:15:03,519 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting fd044c8cc1164bd0947b63e39dd84320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733793301108 2024-12-10T01:15:03,520 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb55cc8dd74492285027772b10d5a13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301043 2024-12-10T01:15:03,520 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c7466c009c2549c7b3136592be297159, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303309 2024-12-10T01:15:03,521 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2206e0cfacf4253967c8757edb2cb14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733793301108 2024-12-10T01:15:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:03,522 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56cbc12f8c94b20a323ce65fdd6c7fc, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303273 2024-12-10T01:15:03,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:03,539 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:03,539 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/9048fb5a2e6141278a204b9ec75d0e37 is 50, key is test_row_0/B:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,563 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:03,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793363566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793363567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793363571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793363571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108ac2f3ef41534506a6cdf4718f803b72_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793303528/Put/seqid=0 2024-12-10T01:15:03,581 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412100ad11b74266a4ed88931f67f14c7adc0_5252bc35c0b22e4dae6eb027696caf9a store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741975_1151 (size=12459) 2024-12-10T01:15:03,585 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412100ad11b74266a4ed88931f67f14c7adc0_5252bc35c0b22e4dae6eb027696caf9a, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,585 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100ad11b74266a4ed88931f67f14c7adc0_5252bc35c0b22e4dae6eb027696caf9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741976_1152 (size=12304) 2024-12-10T01:15:03,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741977_1153 (size=4469) 2024-12-10T01:15:03,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/9048fb5a2e6141278a204b9ec75d0e37 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/9048fb5a2e6141278a204b9ec75d0e37 2024-12-10T01:15:03,621 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/B of 5252bc35c0b22e4dae6eb027696caf9a into 9048fb5a2e6141278a204b9ec75d0e37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:03,621 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,622 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/B, priority=13, startTime=1733793303516; duration=0sec 2024-12-10T01:15:03,622 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:03,622 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:B 2024-12-10T01:15:03,622 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:03,623 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:03,623 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/C is initiating minor compaction (all files) 2024-12-10T01:15:03,623 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/C in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,623 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0f4dc32bf80841c2b921547f95e02c0a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=35.7 K 2024-12-10T01:15:03,624 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4dc32bf80841c2b921547f95e02c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733793301043 2024-12-10T01:15:03,624 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de7590b7c964d678a8636c8a4e8b47a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733793301108 2024-12-10T01:15:03,624 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d4aafa342a2249ccaf8087277f1a991d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303309 2024-12-10T01:15:03,633 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:03,634 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/e00ce5f4a2ca4ef69bb4500ca785f40c is 50, key is test_row_0/C:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:03,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741978_1154 (size=12459) 2024-12-10T01:15:03,655 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/e00ce5f4a2ca4ef69bb4500ca785f40c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/e00ce5f4a2ca4ef69bb4500ca785f40c 2024-12-10T01:15:03,667 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/C of 5252bc35c0b22e4dae6eb027696caf9a into e00ce5f4a2ca4ef69bb4500ca785f40c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:03,667 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:03,668 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/C, priority=13, startTime=1733793303516; duration=0sec 2024-12-10T01:15:03,668 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:03,668 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:C 2024-12-10T01:15:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793363680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793363680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793363680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793363680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793363719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:03,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793363890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793363890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793363890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793363890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:03,997 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:04,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108ac2f3ef41534506a6cdf4718f803b72_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108ac2f3ef41534506a6cdf4718f803b72_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:04,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6465f7cc33764499ad9c0e1b08f6177a, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:04,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6465f7cc33764499ad9c0e1b08f6177a is 175, key is test_row_0/A:col10/1733793303528/Put/seqid=0 2024-12-10T01:15:04,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741979_1155 (size=31105) 2024-12-10T01:15:04,014 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#A#compaction#127 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:04,015 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f0926f92a8a8401a8c0d579b735d47e1 is 175, key is test_row_0/A:col10/1733793303348/Put/seqid=0 2024-12-10T01:15:04,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741980_1156 (size=31413) 2024-12-10T01:15:04,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,027 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f0926f92a8a8401a8c0d579b735d47e1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1 2024-12-10T01:15:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793364029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,036 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/A of 5252bc35c0b22e4dae6eb027696caf9a into f0926f92a8a8401a8c0d579b735d47e1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:04,036 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:04,036 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/A, priority=13, startTime=1733793303515; duration=0sec 2024-12-10T01:15:04,036 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:04,036 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:A 2024-12-10T01:15:04,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793364200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793364200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793364200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793364200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:04,412 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6465f7cc33764499ad9c0e1b08f6177a 2024-12-10T01:15:04,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/03b5ac74cdc14e71ae8328916ba43a7c is 50, key is test_row_0/B:col10/1733793303528/Put/seqid=0 2024-12-10T01:15:04,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741981_1157 (size=12151) 2024-12-10T01:15:04,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793364539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793364709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793364709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793364709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793364719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:04,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/03b5ac74cdc14e71ae8328916ba43a7c 2024-12-10T01:15:04,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9c82ce70a60c46ae8521d7aade962ca8 is 50, key is test_row_0/C:col10/1733793303528/Put/seqid=0 2024-12-10T01:15:04,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741982_1158 (size=12151) 2024-12-10T01:15:04,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:04,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:05,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9c82ce70a60c46ae8521d7aade962ca8 2024-12-10T01:15:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6465f7cc33764499ad9c0e1b08f6177a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a 2024-12-10T01:15:05,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a, entries=150, sequenceid=187, filesize=30.4 K 2024-12-10T01:15:05,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/03b5ac74cdc14e71ae8328916ba43a7c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c 2024-12-10T01:15:05,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:05,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:05,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c, entries=150, sequenceid=187, filesize=11.9 K 2024-12-10T01:15:05,257 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T01:15:05,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9c82ce70a60c46ae8521d7aade962ca8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8 2024-12-10T01:15:05,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8, entries=150, sequenceid=187, filesize=11.9 K 2024-12-10T01:15:05,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 5252bc35c0b22e4dae6eb027696caf9a in 1732ms, sequenceid=187, compaction requested=false 2024-12-10T01:15:05,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:05,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:05,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121019d5f191968441b391bd644477b9f7c9_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793303567/Put/seqid=0 2024-12-10T01:15:05,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741983_1159 (size=12304) 2024-12-10T01:15:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:05,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:05,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793305587, max timestamp=1733793305614Num uniques 859; 2024-12-10T01:15:05,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793305587, max timestamp=1733793305614Num uniques 859; 2024-12-10T01:15:05,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:05,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:05,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793305587, max timestamp=1733793305614Num uniques 859; 2024-12-10T01:15:05,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:05,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793365685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793365719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793365719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793365719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,722 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:05,723 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,723 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793365740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,742 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793365799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:05,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:05,834 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121019d5f191968441b391bd644477b9f7c9_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019d5f191968441b391bd644477b9f7c9_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:05,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1e11f52f53245a08e2bea670b9d9d7b, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:05,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1e11f52f53245a08e2bea670b9d9d7b is 175, key is test_row_0/A:col10/1733793303567/Put/seqid=0 2024-12-10T01:15:05,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741984_1160 (size=31105) 2024-12-10T01:15:06,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793366010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:06,240 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1e11f52f53245a08e2bea670b9d9d7b 2024-12-10T01:15:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4a0c1f4686cf4008a60dd9683e097597 is 50, key is test_row_0/B:col10/1733793303567/Put/seqid=0 2024-12-10T01:15:06,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741985_1161 (size=12151) 2024-12-10T01:15:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793366320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:06,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4a0c1f4686cf4008a60dd9683e097597 2024-12-10T01:15:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/adb82c58dce0493a97b57e7c76843b2f is 50, key is test_row_0/C:col10/1733793303567/Put/seqid=0 2024-12-10T01:15:06,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741986_1162 (size=12151) 2024-12-10T01:15:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793366829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/adb82c58dce0493a97b57e7c76843b2f 2024-12-10T01:15:07,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1e11f52f53245a08e2bea670b9d9d7b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b 2024-12-10T01:15:07,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b, entries=150, sequenceid=200, filesize=30.4 K 2024-12-10T01:15:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4a0c1f4686cf4008a60dd9683e097597 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597 2024-12-10T01:15:07,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597, entries=150, sequenceid=200, filesize=11.9 K 2024-12-10T01:15:07,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/adb82c58dce0493a97b57e7c76843b2f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f 2024-12-10T01:15:07,091 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f, entries=150, sequenceid=200, filesize=11.9 K 2024-12-10T01:15:07,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5252bc35c0b22e4dae6eb027696caf9a in 1687ms, sequenceid=200, compaction requested=true 2024-12-10T01:15:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-10T01:15:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-10T01:15:07,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-10T01:15:07,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8400 sec 2024-12-10T01:15:07,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.8450 sec 2024-12-10T01:15:07,164 INFO [master/fd1a2b66caf8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T01:15:07,164 INFO [master/fd1a2b66caf8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T01:15:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T01:15:07,410 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:07,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-10T01:15:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:07,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:07,413 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:07,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:07,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T01:15:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:07,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T01:15:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:07,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210359848740df141c8b8dffdf6d4665d2a_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793305679/Put/seqid=0 2024-12-10T01:15:07,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741987_1163 (size=12304) 2024-12-10T01:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:07,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793367745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793367746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793367749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793367769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,772 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793367840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,840 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793367859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793367859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793367859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:07,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:07,981 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210359848740df141c8b8dffdf6d4665d2a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210359848740df141c8b8dffdf6d4665d2a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6185b120175d47a7801065cb12121cc3, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6185b120175d47a7801065cb12121cc3 is 175, key is test_row_0/A:col10/1733793305679/Put/seqid=0 2024-12-10T01:15:07,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741988_1164 (size=31105) 2024-12-10T01:15:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793368069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793368069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793368069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793368380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793368380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793368380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6185b120175d47a7801065cb12121cc3 2024-12-10T01:15:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/296b814aff144069a84888633e0518a5 is 50, key is test_row_0/B:col10/1733793305679/Put/seqid=0 2024-12-10T01:15:08,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741989_1165 (size=12151) 2024-12-10T01:15:08,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/296b814aff144069a84888633e0518a5 2024-12-10T01:15:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/0535c4ee28204319b957c4731a7db932 is 50, key is test_row_0/C:col10/1733793305679/Put/seqid=0 2024-12-10T01:15:08,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741990_1166 (size=12151) 2024-12-10T01:15:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:08,862 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/0535c4ee28204319b957c4731a7db932 2024-12-10T01:15:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/6185b120175d47a7801065cb12121cc3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3 2024-12-10T01:15:08,871 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3, entries=150, sequenceid=236, filesize=30.4 K 2024-12-10T01:15:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/296b814aff144069a84888633e0518a5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5 2024-12-10T01:15:08,877 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T01:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/0535c4ee28204319b957c4731a7db932 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932 2024-12-10T01:15:08,882 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T01:15:08,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5252bc35c0b22e4dae6eb027696caf9a in 1318ms, sequenceid=236, compaction requested=true 2024-12-10T01:15:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-10T01:15:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-10T01:15:08,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-10T01:15:08,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-12-10T01:15:08,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4770 sec 2024-12-10T01:15:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:08,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:08,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100aec652394754610a657bd317f8dffee_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:08,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741991_1167 (size=14794) 2024-12-10T01:15:08,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:08,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100aec652394754610a657bd317f8dffee_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100aec652394754610a657bd317f8dffee_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:08,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ee9252370ac941cf8fe24577810bc35f, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:08,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ee9252370ac941cf8fe24577810bc35f is 175, key is test_row_0/A:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741992_1168 (size=39749) 2024-12-10T01:15:08,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793368933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793308898, max timestamp=1733793308935Num uniques 859; 2024-12-10T01:15:08,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:08,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793308898, max timestamp=1733793308935Num uniques 859; 2024-12-10T01:15:08,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:08,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793308898, max timestamp=1733793308935Num uniques 859; 2024-12-10T01:15:08,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793368991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793368993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793369049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793369100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793369100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793369259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793369309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793369309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ee9252370ac941cf8fe24577810bc35f 2024-12-10T01:15:09,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/b5909f0658ce4de097ed5ce6fc5a43f2 is 50, key is test_row_0/B:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:09,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741993_1169 (size=12151) 2024-12-10T01:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T01:15:09,560 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:09,560 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-10T01:15:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:09,562 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:09,563 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:09,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793369570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793369619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793369619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:09,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/b5909f0658ce4de097ed5ce6fc5a43f2 2024-12-10T01:15:09,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/b1f6aa56a3f447698770b3c544319d1b is 50, key is test_row_0/C:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:09,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741994_1170 (size=12151) 2024-12-10T01:15:09,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/b1f6aa56a3f447698770b3c544319d1b 2024-12-10T01:15:09,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ee9252370ac941cf8fe24577810bc35f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f 2024-12-10T01:15:09,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f, entries=200, sequenceid=247, filesize=38.8 K 2024-12-10T01:15:09,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/b5909f0658ce4de097ed5ce6fc5a43f2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2 2024-12-10T01:15:09,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2, entries=150, sequenceid=247, filesize=11.9 K 2024-12-10T01:15:09,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/b1f6aa56a3f447698770b3c544319d1b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b 2024-12-10T01:15:09,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b, entries=150, sequenceid=247, filesize=11.9 K 2024-12-10T01:15:09,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5252bc35c0b22e4dae6eb027696caf9a in 892ms, sequenceid=247, compaction requested=true 2024-12-10T01:15:09,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:09,785 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:09,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T01:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:09,786 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T01:15:09,787 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/B is initiating minor compaction (all files) 2024-12-10T01:15:09,787 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/B in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,787 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164477 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T01:15:09,787 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/A is initiating minor compaction (all files) 2024-12-10T01:15:09,787 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/9048fb5a2e6141278a204b9ec75d0e37, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=59.6 K 2024-12-10T01:15:09,787 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/A in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,787 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=160.6 K 2024-12-10T01:15:09,787 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,787 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f] 2024-12-10T01:15:09,789 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9048fb5a2e6141278a204b9ec75d0e37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303309 2024-12-10T01:15:09,789 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0926f92a8a8401a8c0d579b735d47e1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303309 2024-12-10T01:15:09,789 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b5ac74cdc14e71ae8328916ba43a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733793303434 2024-12-10T01:15:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6465f7cc33764499ad9c0e1b08f6177a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733793303434 2024-12-10T01:15:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0c1f4686cf4008a60dd9683e097597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793303565 2024-12-10T01:15:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e11f52f53245a08e2bea670b9d9d7b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793303565 2024-12-10T01:15:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 296b814aff144069a84888633e0518a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733793305657 2024-12-10T01:15:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6185b120175d47a7801065cb12121cc3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733793305657 2024-12-10T01:15:09,791 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b5909f0658ce4de097ed5ce6fc5a43f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:09,791 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9252370ac941cf8fe24577810bc35f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:09,805 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#B#compaction#141 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:09,805 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/75187caf34ba4fd6981f4acc12e89ed6 is 50, key is test_row_0/B:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:09,807 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:09,810 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121073855391feef4a5397f700cb6919f6c4_5252bc35c0b22e4dae6eb027696caf9a store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:09,813 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121073855391feef4a5397f700cb6919f6c4_5252bc35c0b22e4dae6eb027696caf9a, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:09,813 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121073855391feef4a5397f700cb6919f6c4_5252bc35c0b22e4dae6eb027696caf9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:09,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741995_1171 (size=12629) 2024-12-10T01:15:09,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741996_1172 (size=4469) 2024-12-10T01:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:09,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:09,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105e02a00acdec4346a0eb36636edee399_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793309852/Put/seqid=0 2024-12-10T01:15:09,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:09,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:09,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741997_1173 (size=14994) 2024-12-10T01:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793369886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793369999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793370079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793370130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793370130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:10,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793370210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,226 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/75187caf34ba4fd6981f4acc12e89ed6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/75187caf34ba4fd6981f4acc12e89ed6 2024-12-10T01:15:10,227 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:10,228 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/4e9c002b873b45beac3e120091ebb389 is 175, key is test_row_0/A:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:10,233 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/B of 5252bc35c0b22e4dae6eb027696caf9a into 75187caf34ba4fd6981f4acc12e89ed6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:10,233 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:10,233 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/B, priority=11, startTime=1733793309785; duration=0sec 2024-12-10T01:15:10,233 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:10,233 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:B 2024-12-10T01:15:10,233 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T01:15:10,235 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T01:15:10,235 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/C is initiating minor compaction (all files) 2024-12-10T01:15:10,235 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/C in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741998_1174 (size=31583) 2024-12-10T01:15:10,235 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/e00ce5f4a2ca4ef69bb4500ca785f40c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=59.6 K 2024-12-10T01:15:10,237 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e00ce5f4a2ca4ef69bb4500ca785f40c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733793303309 2024-12-10T01:15:10,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c82ce70a60c46ae8521d7aade962ca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733793303434 2024-12-10T01:15:10,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting adb82c58dce0493a97b57e7c76843b2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793303565 2024-12-10T01:15:10,239 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0535c4ee28204319b957c4731a7db932, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733793305657 2024-12-10T01:15:10,239 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f6aa56a3f447698770b3c544319d1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:10,248 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:10,248 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/c10c2e33a54040498bda918a9c04761c is 50, key is test_row_0/C:col10/1733793307742/Put/seqid=0 2024-12-10T01:15:10,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741999_1175 (size=12629) 2024-12-10T01:15:10,258 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/c10c2e33a54040498bda918a9c04761c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/c10c2e33a54040498bda918a9c04761c 2024-12-10T01:15:10,267 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/C of 5252bc35c0b22e4dae6eb027696caf9a into c10c2e33a54040498bda918a9c04761c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:10,267 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:10,267 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/C, priority=11, startTime=1733793309785; duration=0sec 2024-12-10T01:15:10,267 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:10,267 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:C 2024-12-10T01:15:10,275 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:10,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105e02a00acdec4346a0eb36636edee399_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e02a00acdec4346a0eb36636edee399_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:10,281 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/59ab7fee68f348cea01ee14d08df8ddc, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:10,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/59ab7fee68f348cea01ee14d08df8ddc is 175, key is test_row_0/A:col10/1733793309852/Put/seqid=0 2024-12-10T01:15:10,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742000_1176 (size=39949) 2024-12-10T01:15:10,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793370519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:10,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,644 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/4e9c002b873b45beac3e120091ebb389 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389 2024-12-10T01:15:10,651 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/A of 5252bc35c0b22e4dae6eb027696caf9a into 4e9c002b873b45beac3e120091ebb389(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:10,651 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:10,651 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/A, priority=11, startTime=1733793309784; duration=0sec 2024-12-10T01:15:10,651 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:10,651 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:A 2024-12-10T01:15:10,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/59ab7fee68f348cea01ee14d08df8ddc 2024-12-10T01:15:10,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/97dc4dcf04a34ae4a31e6720d2331eff is 50, key is test_row_0/B:col10/1733793309852/Put/seqid=0 2024-12-10T01:15:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742001_1177 (size=12301) 2024-12-10T01:15:10,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:10,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793371030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:11,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793371089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:11,090 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:11,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/97dc4dcf04a34ae4a31e6720d2331eff 2024-12-10T01:15:11,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/be525c9aad424873a24461ad3c893357 is 50, key is test_row_0/C:col10/1733793309852/Put/seqid=0 2024-12-10T01:15:11,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742002_1178 (size=12301) 2024-12-10T01:15:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793371140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:11,141 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793371149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:11,150 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:11,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:11,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/be525c9aad424873a24461ad3c893357 2024-12-10T01:15:11,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/59ab7fee68f348cea01ee14d08df8ddc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc 2024-12-10T01:15:11,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc, entries=200, sequenceid=284, filesize=39.0 K 2024-12-10T01:15:11,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/97dc4dcf04a34ae4a31e6720d2331eff as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff 2024-12-10T01:15:11,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff, entries=150, sequenceid=284, filesize=12.0 K 2024-12-10T01:15:11,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/be525c9aad424873a24461ad3c893357 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357 2024-12-10T01:15:11,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357, entries=150, sequenceid=284, filesize=12.0 K 2024-12-10T01:15:11,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 5252bc35c0b22e4dae6eb027696caf9a in 1696ms, sequenceid=284, compaction requested=false 2024-12-10T01:15:11,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:11,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T01:15:11,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:11,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:11,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dcf159531cad452a935aa896ad3a9a83_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793309875/Put/seqid=0 2024-12-10T01:15:11,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742003_1179 (size=12454) 2024-12-10T01:15:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793311830, max timestamp=1733793311873Num uniques 859; 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793311830, max timestamp=1733793311873Num uniques 859; 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793311830, max timestamp=1733793311873Num uniques 859; 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793311924, max timestamp=1733793311943Num uniques 1018; 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793311924, max timestamp=1733793311943Num uniques 1018; 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793311924, max timestamp=1733793311943Num uniques 1018; 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:11,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:11,982 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dcf159531cad452a935aa896ad3a9a83_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dcf159531cad452a935aa896ad3a9a83_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:11,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/dd50973096a545baa14db8ab2dddda02, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:11,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/dd50973096a545baa14db8ab2dddda02 is 175, key is test_row_0/A:col10/1733793309875/Put/seqid=0 2024-12-10T01:15:11,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742004_1180 (size=31255) 2024-12-10T01:15:12,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793372032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793372046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:12,049 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793372149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793372360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:12,392 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/dd50973096a545baa14db8ab2dddda02 2024-12-10T01:15:12,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4513f6fa04b446f0bff1b385a0a4f1c8 is 50, key is test_row_0/B:col10/1733793309875/Put/seqid=0 2024-12-10T01:15:12,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742005_1181 (size=12301) 2024-12-10T01:15:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793372669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:12,819 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4513f6fa04b446f0bff1b385a0a4f1c8 2024-12-10T01:15:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9bae219f35a1421789d1f6e2ea13c222 is 50, key is test_row_0/C:col10/1733793309875/Put/seqid=0 2024-12-10T01:15:12,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742006_1182 (size=12301) 2024-12-10T01:15:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793373119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:13,122 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793373150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:13,152 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:13,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793373169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:13,173 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:13,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793373179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:13,230 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9bae219f35a1421789d1f6e2ea13c222 2024-12-10T01:15:13,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/dd50973096a545baa14db8ab2dddda02 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02 2024-12-10T01:15:13,239 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02, entries=150, sequenceid=296, filesize=30.5 K 2024-12-10T01:15:13,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/4513f6fa04b446f0bff1b385a0a4f1c8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8 2024-12-10T01:15:13,245 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8, entries=150, sequenceid=296, filesize=12.0 K 2024-12-10T01:15:13,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/9bae219f35a1421789d1f6e2ea13c222 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222 2024-12-10T01:15:13,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222, entries=150, sequenceid=296, filesize=12.0 K 2024-12-10T01:15:13,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5252bc35c0b22e4dae6eb027696caf9a in 1691ms, sequenceid=296, compaction requested=true 2024-12-10T01:15:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-10T01:15:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-10T01:15:13,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-10T01:15:13,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6910 sec 2024-12-10T01:15:13,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.6970 sec 2024-12-10T01:15:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T01:15:13,729 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:13,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-10T01:15:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:13,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:13,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:13,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:13,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T01:15:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:13,884 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:13,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100cab15cecda04808b9d4c660ae1f962a_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793312006/Put/seqid=0 2024-12-10T01:15:13,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742007_1183 (size=12454) 2024-12-10T01:15:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793374077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793374189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793374189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:14,194 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:14,300 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100cab15cecda04808b9d4c660ae1f962a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cab15cecda04808b9d4c660ae1f962a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f8b48ef6e61f4c47b872ff64336a0e0c, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f8b48ef6e61f4c47b872ff64336a0e0c is 175, key is test_row_0/A:col10/1733793312006/Put/seqid=0 2024-12-10T01:15:14,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742008_1184 (size=31255) 2024-12-10T01:15:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:14,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793374399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:14,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f8b48ef6e61f4c47b872ff64336a0e0c 2024-12-10T01:15:14,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793374709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:14,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/0698b909daf5459a84993ca6a215fff2 is 50, key is test_row_0/B:col10/1733793312006/Put/seqid=0 2024-12-10T01:15:14,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742009_1185 (size=12301) 2024-12-10T01:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:15,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/0698b909daf5459a84993ca6a215fff2 2024-12-10T01:15:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4ea91f636569457eb9b72374a6957dce is 50, key is test_row_0/C:col10/1733793312006/Put/seqid=0 2024-12-10T01:15:15,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742010_1186 (size=12301) 2024-12-10T01:15:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793375220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:15,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4ea91f636569457eb9b72374a6957dce 2024-12-10T01:15:15,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/f8b48ef6e61f4c47b872ff64336a0e0c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c 2024-12-10T01:15:15,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c, entries=150, sequenceid=350, filesize=30.5 K 2024-12-10T01:15:15,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/0698b909daf5459a84993ca6a215fff2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2 2024-12-10T01:15:15,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2, entries=150, sequenceid=350, filesize=12.0 K 2024-12-10T01:15:15,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4ea91f636569457eb9b72374a6957dce as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce 2024-12-10T01:15:15,550 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce, entries=150, sequenceid=350, filesize=12.0 K 2024-12-10T01:15:15,551 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for 5252bc35c0b22e4dae6eb027696caf9a in 1667ms, sequenceid=350, compaction requested=true 2024-12-10T01:15:15,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:15,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-10T01:15:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-10T01:15:15,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T01:15:15,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-12-10T01:15:15,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8240 sec 2024-12-10T01:15:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T01:15:15,880 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:15,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-10T01:15:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:15,882 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:15,882 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:15,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:16,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T01:15:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:16,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121039817721ecf84385995f0f9fed710d4d_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793314060/Put/seqid=0 2024-12-10T01:15:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742011_1187 (size=12454) 2024-12-10T01:15:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:16,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793316247, max timestamp=1733793316257Num uniques 859; 2024-12-10T01:15:16,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:16,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793316247, max timestamp=1733793316257Num uniques 859; 2024-12-10T01:15:16,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793316247, max timestamp=1733793316257Num uniques 859; 2024-12-10T01:15:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:16,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793376298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793316281, max timestamp=1733793316299Num uniques 1018; 2024-12-10T01:15:16,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:16,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793316281, max timestamp=1733793316299Num uniques 1018; 2024-12-10T01:15:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793316281, max timestamp=1733793316299Num uniques 1018; 2024-12-10T01:15:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793376410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793376412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:16,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121039817721ecf84385995f0f9fed710d4d_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039817721ecf84385995f0f9fed710d4d_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:16,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ed26b5be6e3447b3a830e25404f15373, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ed26b5be6e3447b3a830e25404f15373 is 175, key is test_row_0/A:col10/1733793314060/Put/seqid=0 2024-12-10T01:15:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742012_1188 (size=31255) 2024-12-10T01:15:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:16,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793376519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793376519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793376739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793376739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:16,856 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ed26b5be6e3447b3a830e25404f15373 2024-12-10T01:15:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c92375e5917242eaa1f4b708ff7211e5 is 50, key is test_row_0/B:col10/1733793314060/Put/seqid=0 2024-12-10T01:15:16,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742013_1189 (size=12301) 2024-12-10T01:15:16,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c92375e5917242eaa1f4b708ff7211e5 2024-12-10T01:15:16,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/2a0bb5044f8d4d2d9905e83ce7378172 is 50, key is test_row_0/C:col10/1733793314060/Put/seqid=0 2024-12-10T01:15:16,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742014_1190 (size=12301) 2024-12-10T01:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793377049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793377049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34732 deadline: 1733793377140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,142 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34730 deadline: 1733793377169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,173 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34754 deadline: 1733793377219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,220 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 5252bc35c0b22e4dae6eb027696caf9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/2a0bb5044f8d4d2d9905e83ce7378172 2024-12-10T01:15:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ed26b5be6e3447b3a830e25404f15373 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373 2024-12-10T01:15:17,291 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373, entries=150, sequenceid=359, filesize=30.5 K 2024-12-10T01:15:17,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/c92375e5917242eaa1f4b708ff7211e5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5 2024-12-10T01:15:17,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5, entries=150, sequenceid=359, filesize=12.0 K 2024-12-10T01:15:17,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/2a0bb5044f8d4d2d9905e83ce7378172 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172 2024-12-10T01:15:17,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172, entries=150, sequenceid=359, filesize=12.0 K 2024-12-10T01:15:17,302 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5252bc35c0b22e4dae6eb027696caf9a in 1268ms, sequenceid=359, compaction requested=true 2024-12-10T01:15:17,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:17,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:17,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-10T01:15:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-10T01:15:17,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T01:15:17,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-12-10T01:15:17,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4250 sec 2024-12-10T01:15:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:17,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-10T01:15:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:17,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:17,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:17,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:17,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:17,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104d87c0d5baed4975aacdb2c5713c9ed4_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:17,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742015_1191 (size=14994) 2024-12-10T01:15:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793377571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793377577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793377679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793377689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793377889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793377900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:17,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:17,978 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104d87c0d5baed4975aacdb2c5713c9ed4_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104d87c0d5baed4975aacdb2c5713c9ed4_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:17,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/77348a30dc2d4e2596c8a28f6d19ddc0, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:17,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/77348a30dc2d4e2596c8a28f6d19ddc0 is 175, key is test_row_0/A:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:17,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742016_1192 (size=39949) 2024-12-10T01:15:17,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/77348a30dc2d4e2596c8a28f6d19ddc0 2024-12-10T01:15:17,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/7c969236f88d43ac8434cd2ddd1ea487 is 50, key is test_row_0/B:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:17,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742017_1193 (size=12301) 2024-12-10T01:15:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T01:15:18,030 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:18,030 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-10T01:15:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:18,033 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:18,033 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:18,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:18,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793378199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793378209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:18,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:18,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/7c969236f88d43ac8434cd2ddd1ea487 2024-12-10T01:15:18,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4f28d30649934357ba6e9168a7bfd8bb is 50, key is test_row_0/C:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:18,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742018_1194 (size=12301) 2024-12-10T01:15:18,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34780 deadline: 1733793378709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34762 deadline: 1733793378719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:18,762 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-10T01:15:18,762 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:18,762 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,762 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:18,764 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-10T01:15:18,764 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-10T01:15:18,764 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:18,764 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:18,764 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,764 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,764 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,764 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:18,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:18,769 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-10T01:15:18,769 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:18,769 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,769 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:18,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:18,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:18,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4f28d30649934357ba6e9168a7bfd8bb 2024-12-10T01:15:18,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/77348a30dc2d4e2596c8a28f6d19ddc0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0 2024-12-10T01:15:18,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0, entries=200, sequenceid=414, filesize=39.0 K 2024-12-10T01:15:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/7c969236f88d43ac8434cd2ddd1ea487 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487 2024-12-10T01:15:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487, entries=150, sequenceid=414, filesize=12.0 K 2024-12-10T01:15:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/4f28d30649934357ba6e9168a7bfd8bb as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb 2024-12-10T01:15:18,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb, entries=150, sequenceid=414, filesize=12.0 K 2024-12-10T01:15:18,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 5252bc35c0b22e4dae6eb027696caf9a in 1326ms, sequenceid=414, compaction requested=true 2024-12-10T01:15:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:18,887 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-10T01:15:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:18,888 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-10T01:15:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5252bc35c0b22e4dae6eb027696caf9a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:18,889 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 205246 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-10T01:15:18,889 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74134 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-10T01:15:18,889 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/B is initiating minor compaction (all files) 2024-12-10T01:15:18,889 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/A is initiating minor compaction (all files) 2024-12-10T01:15:18,889 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/A in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,889 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/B in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,889 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/75187caf34ba4fd6981f4acc12e89ed6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=72.4 K 2024-12-10T01:15:18,889 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=200.4 K 2024-12-10T01:15:18,890 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0] 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 75187caf34ba4fd6981f4acc12e89ed6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9c002b873b45beac3e120091ebb389, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 97dc4dcf04a34ae4a31e6720d2331eff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733793308976 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ab7fee68f348cea01ee14d08df8ddc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733793308976 2024-12-10T01:15:18,890 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4513f6fa04b446f0bff1b385a0a4f1c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733793309855 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd50973096a545baa14db8ab2dddda02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733793309855 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0698b909daf5459a84993ca6a215fff2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733793312006 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b48ef6e61f4c47b872ff64336a0e0c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733793312006 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c92375e5917242eaa1f4b708ff7211e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733793314060 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed26b5be6e3447b3a830e25404f15373, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733793314060 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c969236f88d43ac8434cd2ddd1ea487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733793316368 2024-12-10T01:15:18,891 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77348a30dc2d4e2596c8a28f6d19ddc0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733793316368 2024-12-10T01:15:18,903 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#B#compaction#159 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:18,904 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/ada883ca73304cd19336f07002d0063c is 50, key is test_row_0/B:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:18,905 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:18,907 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121007efc9a10068403aa2f8b0be80d4038e_5252bc35c0b22e4dae6eb027696caf9a store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:18,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742019_1195 (size=12983) 2024-12-10T01:15:18,920 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/ada883ca73304cd19336f07002d0063c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ada883ca73304cd19336f07002d0063c 2024-12-10T01:15:18,926 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/B of 5252bc35c0b22e4dae6eb027696caf9a into ada883ca73304cd19336f07002d0063c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:18,926 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:18,926 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/B, priority=10, startTime=1733793318887; duration=0sec 2024-12-10T01:15:18,926 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:18,926 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:B 2024-12-10T01:15:18,926 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-10T01:15:18,928 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74134 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-10T01:15:18,928 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 5252bc35c0b22e4dae6eb027696caf9a/C is initiating minor compaction (all files) 2024-12-10T01:15:18,928 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5252bc35c0b22e4dae6eb027696caf9a/C in TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,928 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/c10c2e33a54040498bda918a9c04761c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp, totalSize=72.4 K 2024-12-10T01:15:18,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c10c2e33a54040498bda918a9c04761c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733793307742 2024-12-10T01:15:18,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting be525c9aad424873a24461ad3c893357, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733793308976 2024-12-10T01:15:18,930 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bae219f35a1421789d1f6e2ea13c222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733793309855 2024-12-10T01:15:18,930 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea91f636569457eb9b72374a6957dce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733793312006 2024-12-10T01:15:18,930 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0bb5044f8d4d2d9905e83ce7378172, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733793314060 2024-12-10T01:15:18,931 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f28d30649934357ba6e9168a7bfd8bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733793316368 2024-12-10T01:15:18,935 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121007efc9a10068403aa2f8b0be80d4038e_5252bc35c0b22e4dae6eb027696caf9a, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:18,936 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121007efc9a10068403aa2f8b0be80d4038e_5252bc35c0b22e4dae6eb027696caf9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742020_1196 (size=4469) 2024-12-10T01:15:18,940 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#A#compaction#160 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:18,941 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ccc90a20f7bf43458cf6620ca8889cf9 is 175, key is test_row_0/A:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742021_1197 (size=31937) 2024-12-10T01:15:18,948 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5252bc35c0b22e4dae6eb027696caf9a#C#compaction#161 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:18,949 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/b9bab173d08d47e9b1128b94428fc61e is 50, key is test_row_0/C:col10/1733793317560/Put/seqid=0 2024-12-10T01:15:18,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:18,951 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:18,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742022_1198 (size=12983) 2024-12-10T01:15:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c1c6c8c203ad4ab6837c67970fef1ec0_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793317575/Put/seqid=0 2024-12-10T01:15:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742023_1199 (size=12454) 2024-12-10T01:15:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:19,357 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/ccc90a20f7bf43458cf6620ca8889cf9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccc90a20f7bf43458cf6620ca8889cf9 2024-12-10T01:15:19,360 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/b9bab173d08d47e9b1128b94428fc61e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b9bab173d08d47e9b1128b94428fc61e 2024-12-10T01:15:19,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:19,364 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/A of 5252bc35c0b22e4dae6eb027696caf9a into ccc90a20f7bf43458cf6620ca8889cf9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:19,364 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:19,364 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/A, priority=10, startTime=1733793318887; duration=0sec 2024-12-10T01:15:19,364 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:19,364 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:A 2024-12-10T01:15:19,366 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c1c6c8c203ad4ab6837c67970fef1ec0_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c1c6c8c203ad4ab6837c67970fef1ec0_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:19,367 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5252bc35c0b22e4dae6eb027696caf9a/C of 5252bc35c0b22e4dae6eb027696caf9a into b9bab173d08d47e9b1128b94428fc61e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:19,367 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:19,367 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., storeName=5252bc35c0b22e4dae6eb027696caf9a/C, priority=10, startTime=1733793318888; duration=0sec 2024-12-10T01:15:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cc9c7d7792964a3fa5d2af66ed80ab46, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:19,367 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:19,367 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5252bc35c0b22e4dae6eb027696caf9a:C 2024-12-10T01:15:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cc9c7d7792964a3fa5d2af66ed80ab46 is 175, key is test_row_0/A:col10/1733793317575/Put/seqid=0 2024-12-10T01:15:19,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742024_1200 (size=31255) 2024-12-10T01:15:19,724 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-12-10T01:15:19,724 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:19,724 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:19,724 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:19,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. as already flushing 2024-12-10T01:15:19,741 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-12-10T01:15:19,741 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:19,741 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:19,741 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:19,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:19,771 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cc9c7d7792964a3fa5d2af66ed80ab46 2024-12-10T01:15:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d329f45bd0394dd1beb38dd0cbe0060e is 50, key is test_row_0/B:col10/1733793317575/Put/seqid=0 2024-12-10T01:15:19,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742025_1201 (size=12301) 2024-12-10T01:15:20,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d329f45bd0394dd1beb38dd0cbe0060e 2024-12-10T01:15:20,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/46cc2457d2fb4786ac57683f397c913a is 50, key is test_row_0/C:col10/1733793317575/Put/seqid=0 2024-12-10T01:15:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742026_1202 (size=12301) 2024-12-10T01:15:20,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/46cc2457d2fb4786ac57683f397c913a 2024-12-10T01:15:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/cc9c7d7792964a3fa5d2af66ed80ab46 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cc9c7d7792964a3fa5d2af66ed80ab46 2024-12-10T01:15:20,613 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cc9c7d7792964a3fa5d2af66ed80ab46, entries=150, sequenceid=423, filesize=30.5 K 2024-12-10T01:15:20,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d329f45bd0394dd1beb38dd0cbe0060e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d329f45bd0394dd1beb38dd0cbe0060e 2024-12-10T01:15:20,619 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d329f45bd0394dd1beb38dd0cbe0060e, entries=150, sequenceid=423, filesize=12.0 K 2024-12-10T01:15:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/46cc2457d2fb4786ac57683f397c913a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/46cc2457d2fb4786ac57683f397c913a 2024-12-10T01:15:20,624 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/46cc2457d2fb4786ac57683f397c913a, entries=150, sequenceid=423, filesize=12.0 K 2024-12-10T01:15:20,625 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=13.42 KB/13740 for 5252bc35c0b22e4dae6eb027696caf9a in 1674ms, sequenceid=423, compaction requested=false 2024-12-10T01:15:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5252bc35c0b22e4dae6eb027696caf9a: 2024-12-10T01:15:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-10T01:15:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-10T01:15:20,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T01:15:20,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5940 sec 2024-12-10T01:15:20,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5990 sec 2024-12-10T01:15:21,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T01:15:22,200 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:27,241 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-12-10T01:15:27,241 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:27,241 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,241 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:27,251 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-12-10T01:15:27,251 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:27,251 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,251 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:27,301 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-12-10T01:15:27,301 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:27,301 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,301 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3758 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3524 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1545 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4634 rows 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1559 2024-12-10T01:15:27,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4676 rows 2024-12-10T01:15:27,301 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:15:27,301 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:15:27,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:27,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:27,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:15:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:27,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793327305"}]},"ts":"1733793327305"} 2024-12-10T01:15:27,307 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:15:27,307 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:15:27,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:15:27,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, UNASSIGN}] 2024-12-10T01:15:27,310 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, UNASSIGN 2024-12-10T01:15:27,311 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:27,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, UNASSIGN because future has completed 2024-12-10T01:15:27,313 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:15:27,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:27,466 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 5252bc35c0b22e4dae6eb027696caf9a, disabling compactions & flushes 2024-12-10T01:15:27,466 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. after waiting 0 ms 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:27,466 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 5252bc35c0b22e4dae6eb027696caf9a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:15:27,466 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=A 2024-12-10T01:15:27,467 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:27,467 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=B 2024-12-10T01:15:27,467 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:27,467 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5252bc35c0b22e4dae6eb027696caf9a, store=C 2024-12-10T01:15:27,467 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:27,474 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a15780074b3a42ecb186d5aa5d070228_5252bc35c0b22e4dae6eb027696caf9a is 50, key is test_row_0/A:col10/1733793327240/Put/seqid=0 2024-12-10T01:15:27,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742027_1203 (size=12454) 2024-12-10T01:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:27,883 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:27,892 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a15780074b3a42ecb186d5aa5d070228_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a15780074b3a42ecb186d5aa5d070228_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:27,893 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1671ce06e3a4b8db8b221b7bd472535, store: [table=TestAcidGuarantees family=A region=5252bc35c0b22e4dae6eb027696caf9a] 2024-12-10T01:15:27,894 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1671ce06e3a4b8db8b221b7bd472535 is 175, key is test_row_0/A:col10/1733793327240/Put/seqid=0 2024-12-10T01:15:27,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742028_1204 (size=31255) 2024-12-10T01:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:28,298 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1671ce06e3a4b8db8b221b7bd472535 2024-12-10T01:15:28,307 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d5bd5794a1794dd58caa033e3aa18d3b is 50, key is test_row_0/B:col10/1733793327240/Put/seqid=0 2024-12-10T01:15:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742029_1205 (size=12301) 2024-12-10T01:15:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:28,711 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d5bd5794a1794dd58caa033e3aa18d3b 2024-12-10T01:15:28,722 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/dcb39851ee6f4f258e723899fa497362 is 50, key is test_row_0/C:col10/1733793327240/Put/seqid=0 2024-12-10T01:15:28,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742030_1206 (size=12301) 2024-12-10T01:15:29,128 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/dcb39851ee6f4f258e723899fa497362 2024-12-10T01:15:29,138 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/A/e1671ce06e3a4b8db8b221b7bd472535 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1671ce06e3a4b8db8b221b7bd472535 2024-12-10T01:15:29,142 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1671ce06e3a4b8db8b221b7bd472535, entries=150, sequenceid=433, filesize=30.5 K 2024-12-10T01:15:29,143 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/B/d5bd5794a1794dd58caa033e3aa18d3b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d5bd5794a1794dd58caa033e3aa18d3b 2024-12-10T01:15:29,146 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d5bd5794a1794dd58caa033e3aa18d3b, entries=150, sequenceid=433, filesize=12.0 K 2024-12-10T01:15:29,147 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/.tmp/C/dcb39851ee6f4f258e723899fa497362 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/dcb39851ee6f4f258e723899fa497362 2024-12-10T01:15:29,150 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/dcb39851ee6f4f258e723899fa497362, entries=150, sequenceid=433, filesize=12.0 K 2024-12-10T01:15:29,151 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5252bc35c0b22e4dae6eb027696caf9a in 1685ms, sequenceid=433, compaction requested=true 2024-12-10T01:15:29,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0] to archive 2024-12-10T01:15:29,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:29,154 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b9f0aedb91d945e4b7f0cc4c009e26c5 2024-12-10T01:15:29,155 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b5fde365447d4c968132e4c474f63235 2024-12-10T01:15:29,155 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/be26520cb80246499daaf6648d91dc6a 2024-12-10T01:15:29,155 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/856b99e687714c9c89a535652412dbbf 2024-12-10T01:15:29,155 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/efc80766eead4a2597b329014120b355 2024-12-10T01:15:29,155 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cb039948945241e6a032ccee7ac0983d 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccb55cc8dd74492285027772b10d5a13 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/b2206e0cfacf4253967c8757edb2cb14 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1e11f52f53245a08e2bea670b9d9d7b 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6185b120175d47a7801065cb12121cc3 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/a56cbc12f8c94b20a323ce65fdd6c7fc 2024-12-10T01:15:29,156 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/6465f7cc33764499ad9c0e1b08f6177a 2024-12-10T01:15:29,157 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f0926f92a8a8401a8c0d579b735d47e1 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ee9252370ac941cf8fe24577810bc35f 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/59ab7fee68f348cea01ee14d08df8ddc 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/4e9c002b873b45beac3e120091ebb389 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/77348a30dc2d4e2596c8a28f6d19ddc0 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/f8b48ef6e61f4c47b872ff64336a0e0c 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ed26b5be6e3447b3a830e25404f15373 2024-12-10T01:15:29,158 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/dd50973096a545baa14db8ab2dddda02 2024-12-10T01:15:29,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0fc604463aa4440ea781db898c65dac4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7af805d433dc493b94d816a9949a44a9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/9048fb5a2e6141278a204b9ec75d0e37, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/75187caf34ba4fd6981f4acc12e89ed6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487] to archive 2024-12-10T01:15:29,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:29,162 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/acadfebbffb44da5ad5f414b55b118df 2024-12-10T01:15:29,162 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/fd044c8cc1164bd0947b63e39dd84320 2024-12-10T01:15:29,162 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0fc604463aa4440ea781db898c65dac4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0fc604463aa4440ea781db898c65dac4 2024-12-10T01:15:29,162 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ae63abac07134511b9c8a13f3da06ec1 2024-12-10T01:15:29,163 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/a479a58dfe3244369a761db6450d65a7 2024-12-10T01:15:29,163 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/22bc3d8391de49738edfb711d396633a 2024-12-10T01:15:29,163 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7af805d433dc493b94d816a9949a44a9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7af805d433dc493b94d816a9949a44a9 2024-12-10T01:15:29,163 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/79a04b733aa340af9613b62192ac31a3 2024-12-10T01:15:29,164 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/296b814aff144069a84888633e0518a5 2024-12-10T01:15:29,164 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c7466c009c2549c7b3136592be297159 2024-12-10T01:15:29,164 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/9048fb5a2e6141278a204b9ec75d0e37 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/9048fb5a2e6141278a204b9ec75d0e37 2024-12-10T01:15:29,164 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/03b5ac74cdc14e71ae8328916ba43a7c 2024-12-10T01:15:29,166 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/b5909f0658ce4de097ed5ce6fc5a43f2 2024-12-10T01:15:29,166 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4a0c1f4686cf4008a60dd9683e097597 2024-12-10T01:15:29,166 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/75187caf34ba4fd6981f4acc12e89ed6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/75187caf34ba4fd6981f4acc12e89ed6 2024-12-10T01:15:29,166 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/97dc4dcf04a34ae4a31e6720d2331eff 2024-12-10T01:15:29,167 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/4513f6fa04b446f0bff1b385a0a4f1c8 2024-12-10T01:15:29,167 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/c92375e5917242eaa1f4b708ff7211e5 2024-12-10T01:15:29,167 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/7c969236f88d43ac8434cd2ddd1ea487 2024-12-10T01:15:29,167 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/0698b909daf5459a84993ca6a215fff2 2024-12-10T01:15:29,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/74b6ed257dd2449a9b8311cd566a10ab, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0f4dc32bf80841c2b921547f95e02c0a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/e00ce5f4a2ca4ef69bb4500ca785f40c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/c10c2e33a54040498bda918a9c04761c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb] to archive 2024-12-10T01:15:29,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/32811c65e8354a1798c1dc0d67f15e58 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/bd8bd9477b7d415b8f8c03aadaa5261e 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/3de7590b7c964d678a8636c8a4e8b47a 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/7417640dc7024cd0bc776dc92362ab50 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/15d40c1ce3be4b809d9a6ed6af14cc55 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0f4dc32bf80841c2b921547f95e02c0a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0f4dc32bf80841c2b921547f95e02c0a 2024-12-10T01:15:29,171 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4d0452b8586742b99a9df72fe07083fa 2024-12-10T01:15:29,172 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/adb82c58dce0493a97b57e7c76843b2f 2024-12-10T01:15:29,172 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/d4aafa342a2249ccaf8087277f1a991d 2024-12-10T01:15:29,173 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9c82ce70a60c46ae8521d7aade962ca8 2024-12-10T01:15:29,173 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/e00ce5f4a2ca4ef69bb4500ca785f40c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/e00ce5f4a2ca4ef69bb4500ca785f40c 2024-12-10T01:15:29,173 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/0535c4ee28204319b957c4731a7db932 2024-12-10T01:15:29,173 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b1f6aa56a3f447698770b3c544319d1b 2024-12-10T01:15:29,173 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/c10c2e33a54040498bda918a9c04761c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/c10c2e33a54040498bda918a9c04761c 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/74b6ed257dd2449a9b8311cd566a10ab to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/74b6ed257dd2449a9b8311cd566a10ab 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/be525c9aad424873a24461ad3c893357 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/9bae219f35a1421789d1f6e2ea13c222 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/2a0bb5044f8d4d2d9905e83ce7378172 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4ea91f636569457eb9b72374a6957dce 2024-12-10T01:15:29,174 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/4f28d30649934357ba6e9168a7bfd8bb 2024-12-10T01:15:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-12-10T01:15:29,178 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. 2024-12-10T01:15:29,179 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 5252bc35c0b22e4dae6eb027696caf9a: Waiting for close lock at 1733793327466Running coprocessor pre-close hooks at 1733793327466Disabling compacts and flushes for region at 1733793327466Disabling writes for close at 1733793327466Obtaining lock to block concurrent updates at 1733793327466Preparing flush snapshotting stores in 5252bc35c0b22e4dae6eb027696caf9a at 1733793327466Finished memstore snapshotting TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733793327467 (+1 ms)Flushing stores of TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a. at 1733793327467Flushing 5252bc35c0b22e4dae6eb027696caf9a/A: creating writer at 1733793327467Flushing mob file 5252bc35c0b22e4dae6eb027696caf9a/A: appending metadata at 1733793327474 (+7 ms)Flushing mob file 5252bc35c0b22e4dae6eb027696caf9a/A: closing flushed file at 1733793327474Flushing 5252bc35c0b22e4dae6eb027696caf9a/A: appending metadata at 1733793327893 (+419 ms)Flushing 5252bc35c0b22e4dae6eb027696caf9a/A: closing flushed file at 1733793327893Flushing 5252bc35c0b22e4dae6eb027696caf9a/B: creating writer at 1733793328303 (+410 ms)Flushing 5252bc35c0b22e4dae6eb027696caf9a/B: appending metadata at 1733793328306 (+3 ms)Flushing 5252bc35c0b22e4dae6eb027696caf9a/B: closing flushed file at 1733793328306Flushing 5252bc35c0b22e4dae6eb027696caf9a/C: creating writer at 1733793328717 (+411 ms)Flushing 5252bc35c0b22e4dae6eb027696caf9a/C: appending metadata at 1733793328721 (+4 ms)Flushing 5252bc35c0b22e4dae6eb027696caf9a/C: closing flushed file at 1733793328721Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a7911e: reopening flushed file at 1733793329138 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b1789b5: reopening flushed file at 1733793329142 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d314e7: reopening flushed file at 1733793329146 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5252bc35c0b22e4dae6eb027696caf9a in 1685ms, sequenceid=433, compaction requested=true at 1733793329151 (+5 ms)Writing region close event to WAL at 1733793329175 (+24 ms)Running coprocessor post-close hooks at 1733793329178 (+3 ms)Closed at 1733793329178 2024-12-10T01:15:29,180 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,181 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=5252bc35c0b22e4dae6eb027696caf9a, regionState=CLOSED 2024-12-10T01:15:29,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:29,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-10T01:15:29,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 5252bc35c0b22e4dae6eb027696caf9a, server=fd1a2b66caf8,38925,1733793262612 in 1.8700 sec 2024-12-10T01:15:29,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-12-10T01:15:29,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5252bc35c0b22e4dae6eb027696caf9a, UNASSIGN in 1.8750 sec 2024-12-10T01:15:29,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-10T01:15:29,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8780 sec 2024-12-10T01:15:29,189 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793329189"}]},"ts":"1733793329189"} 2024-12-10T01:15:29,190 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:15:29,190 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:15:29,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8890 sec 2024-12-10T01:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T01:15:29,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:29,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:15:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,457 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T01:15:29,458 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,461 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,467 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits] 2024-12-10T01:15:29,471 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cc9c7d7792964a3fa5d2af66ed80ab46 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/cc9c7d7792964a3fa5d2af66ed80ab46 2024-12-10T01:15:29,471 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1671ce06e3a4b8db8b221b7bd472535 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/e1671ce06e3a4b8db8b221b7bd472535 2024-12-10T01:15:29,471 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccc90a20f7bf43458cf6620ca8889cf9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/A/ccc90a20f7bf43458cf6620ca8889cf9 2024-12-10T01:15:29,475 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d329f45bd0394dd1beb38dd0cbe0060e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d329f45bd0394dd1beb38dd0cbe0060e 2024-12-10T01:15:29,475 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ada883ca73304cd19336f07002d0063c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/ada883ca73304cd19336f07002d0063c 2024-12-10T01:15:29,475 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d5bd5794a1794dd58caa033e3aa18d3b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/B/d5bd5794a1794dd58caa033e3aa18d3b 2024-12-10T01:15:29,478 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/46cc2457d2fb4786ac57683f397c913a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/46cc2457d2fb4786ac57683f397c913a 2024-12-10T01:15:29,478 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b9bab173d08d47e9b1128b94428fc61e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/b9bab173d08d47e9b1128b94428fc61e 2024-12-10T01:15:29,479 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/dcb39851ee6f4f258e723899fa497362 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/C/dcb39851ee6f4f258e723899fa497362 2024-12-10T01:15:29,482 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits/436.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a/recovered.edits/436.seqid 2024-12-10T01:15:29,482 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:15:29,483 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:15:29,483 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101f0ab4fce1d74d7aa6218ce3c786db79_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101f0ab4fce1d74d7aa6218ce3c786db79_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019d5f191968441b391bd644477b9f7c9_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019d5f191968441b391bd644477b9f7c9_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cab15cecda04808b9d4c660ae1f962a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cab15cecda04808b9d4c660ae1f962a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100aec652394754610a657bd317f8dffee_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100aec652394754610a657bd317f8dffee_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c78bf0293864719af98e0a055121b8a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c78bf0293864719af98e0a055121b8a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039817721ecf84385995f0f9fed710d4d_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039817721ecf84385995f0f9fed710d4d_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,493 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210359848740df141c8b8dffdf6d4665d2a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210359848740df141c8b8dffdf6d4665d2a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,494 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104d87c0d5baed4975aacdb2c5713c9ed4_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104d87c0d5baed4975aacdb2c5713c9ed4_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108ac2f3ef41534506a6cdf4718f803b72_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108ac2f3ef41534506a6cdf4718f803b72_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a15780074b3a42ecb186d5aa5d070228_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a15780074b3a42ecb186d5aa5d070228_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108cbbf2a8db004683a3facdc9f3c9c0ee_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108cbbf2a8db004683a3facdc9f3c9c0ee_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e02a00acdec4346a0eb36636edee399_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e02a00acdec4346a0eb36636edee399_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a5d51253e3764fa9ad6758df93de3abe_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a5d51253e3764fa9ad6758df93de3abe_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5a241ac644c4d05b960b310cf74dc8a_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5a241ac644c4d05b960b310cf74dc8a_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c1c6c8c203ad4ab6837c67970fef1ec0_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c1c6c8c203ad4ab6837c67970fef1ec0_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,495 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db5bfbb949fd48b0b0f67bd3cc27f305_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db5bfbb949fd48b0b0f67bd3cc27f305_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,496 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dcf159531cad452a935aa896ad3a9a83_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dcf159531cad452a935aa896ad3a9a83_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,496 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb3c4277413d4d228c2c9c8ea1fde210_5252bc35c0b22e4dae6eb027696caf9a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb3c4277413d4d228c2c9c8ea1fde210_5252bc35c0b22e4dae6eb027696caf9a 2024-12-10T01:15:29,496 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:15:29,499 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,501 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:15:29,503 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:15:29,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:15:29,505 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793329504"}]},"ts":"9223372036854775807"} 2024-12-10T01:15:29,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:15:29,507 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5252bc35c0b22e4dae6eb027696caf9a, NAME => 'TestAcidGuarantees,,1733793295513.5252bc35c0b22e4dae6eb027696caf9a.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:15:29,507 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:15:29,507 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793329507"}]},"ts":"9223372036854775807"} 2024-12-10T01:15:29,509 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:15:29,510 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-10T01:15:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T01:15:29,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:15:29,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:29,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255 (was 255), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=8186 (was 7328) - AvailableMemoryMB LEAK? - 2024-12-10T01:15:29,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=255, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=8186 2024-12-10T01:15:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:15:29,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:15:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:29,585 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:15:29,585 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:29,585 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-12-10T01:15:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-10T01:15:29,586 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:15:29,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742031_1207 (size=1061) 2024-12-10T01:15:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-10T01:15:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-10T01:15:30,000 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:15:30,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742032_1208 (size=53) 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f61d17f4c75f3711af1be3dee84b7c70, disabling compactions & flushes 2024-12-10T01:15:30,009 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. after waiting 0 ms 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,009 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f61d17f4c75f3711af1be3dee84b7c70: Waiting for close lock at 1733793330009Disabling compacts and flushes for region at 1733793330009Disabling writes for close at 1733793330009Writing region close event to WAL at 1733793330009Closed at 1733793330009 2024-12-10T01:15:30,011 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:15:30,011 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793330011"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793330011"}]},"ts":"1733793330011"} 2024-12-10T01:15:30,013 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:15:30,014 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:15:30,014 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793330014"}]},"ts":"1733793330014"} 2024-12-10T01:15:30,016 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:15:30,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, ASSIGN}] 2024-12-10T01:15:30,018 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, ASSIGN 2024-12-10T01:15:30,019 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:15:30,170 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f61d17f4c75f3711af1be3dee84b7c70, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, ASSIGN because future has completed 2024-12-10T01:15:30,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-10T01:15:30,330 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,331 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:15:30,331 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,331 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:15:30,331 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,332 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,334 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,336 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:30,337 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f61d17f4c75f3711af1be3dee84b7c70 columnFamilyName A 2024-12-10T01:15:30,337 DEBUG [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:30,338 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(327): Store=f61d17f4c75f3711af1be3dee84b7c70/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:30,338 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,340 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:30,340 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f61d17f4c75f3711af1be3dee84b7c70 columnFamilyName B 2024-12-10T01:15:30,341 DEBUG [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:30,342 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(327): Store=f61d17f4c75f3711af1be3dee84b7c70/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:30,342 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,344 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:30,344 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f61d17f4c75f3711af1be3dee84b7c70 columnFamilyName C 2024-12-10T01:15:30,344 DEBUG [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:30,345 INFO [StoreOpener-f61d17f4c75f3711af1be3dee84b7c70-1 {}] regionserver.HStore(327): Store=f61d17f4c75f3711af1be3dee84b7c70/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:30,345 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,345 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,346 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,346 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,347 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,347 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,348 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:15:30,349 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,352 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:15:30,352 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f61d17f4c75f3711af1be3dee84b7c70; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67701585, jitterRate=0.008832231163978577}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:15:30,352 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,353 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f61d17f4c75f3711af1be3dee84b7c70: Running coprocessor pre-open hook at 1733793330332Writing region info on filesystem at 1733793330332Initializing all the Stores at 1733793330333 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793330333Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793330334 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793330334Cleaning up temporary data from old regions at 1733793330347 (+13 ms)Running coprocessor post-open hooks at 1733793330352 (+5 ms)Region opened successfully at 1733793330353 (+1 ms) 2024-12-10T01:15:30,354 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., pid=55, masterSystemTime=1733793330326 2024-12-10T01:15:30,356 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,356 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,357 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f61d17f4c75f3711af1be3dee84b7c70, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:30,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-10T01:15:30,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 in 187 msec 2024-12-10T01:15:30,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-10T01:15:30,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, ASSIGN in 343 msec 2024-12-10T01:15:30,362 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:15:30,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793330362"}]},"ts":"1733793330362"} 2024-12-10T01:15:30,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:15:30,364 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:15:30,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 782 msec 2024-12-10T01:15:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-10T01:15:30,720 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:30,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38088ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbee45b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b2c10b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,725 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,729 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a4fd727 2024-12-10T01:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:15:30,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:15:30,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d39793e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9009c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,734 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50acf639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bf3f2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357c1c58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480d5da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb3d51a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d86e07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1d32e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b86b0bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d1d386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713a25c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12af8fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2523e9f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,746 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a9099c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a32328a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e4e6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ea540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2512d3d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6def0813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d76807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fc184c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,754 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e05d4f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,755 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a99584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f755b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,757 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ebed47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,758 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683ddcda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72411e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:30,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:30,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621c608a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:30,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45148463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:30,764 DEBUG [Thread-1003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,765 DEBUG [Thread-996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-10T01:15:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,766 DEBUG [Thread-990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,766 DEBUG [Thread-998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:30,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,767 DEBUG [Thread-1007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,767 DEBUG [Thread-998 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,768 DEBUG [Thread-998 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,768 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:30,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:30,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:30,777 DEBUG [Thread-1009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,777 DEBUG [Thread-992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:30,778 DEBUG [Thread-1005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,779 DEBUG [Thread-1005 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,779 DEBUG [Thread-1005 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,780 DEBUG [Thread-994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,783 DEBUG [Thread-1001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:15:30,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/c57e9611cdd840fd8a11584243b00998 is 50, key is test_row_0/A:col10/1733793330774/Put/seqid=0 2024-12-10T01:15:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793390808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793390810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793330803, max timestamp=1733793330812Num uniques 859; 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793330803, max timestamp=1733793330812Num uniques 859; 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793330803, max timestamp=1733793330812Num uniques 859; 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:30,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742033_1209 (size=12001) 2024-12-10T01:15:30,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/c57e9611cdd840fd8a11584243b00998 2024-12-10T01:15:30,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793390851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/bcce328365354897a5ca445d2ba8cdb0 is 50, key is test_row_0/B:col10/1733793330774/Put/seqid=0 2024-12-10T01:15:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793390853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793390851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742034_1210 (size=12001) 2024-12-10T01:15:30,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/bcce328365354897a5ca445d2ba8cdb0 2024-12-10T01:15:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:30,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a435da89f77641c4b8c1dcc16a308ea9 is 50, key is test_row_0/C:col10/1733793330774/Put/seqid=0 2024-12-10T01:15:30,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793390919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793390920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-10T01:15:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:30,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:30,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742035_1211 (size=12001) 2024-12-10T01:15:30,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793390959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793390969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:30,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793390969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-10T01:15:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:31,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793391130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793391130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793391169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793391180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793391180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-10T01:15:31,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:31,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a435da89f77641c4b8c1dcc16a308ea9 2024-12-10T01:15:31,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/c57e9611cdd840fd8a11584243b00998 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998 2024-12-10T01:15:31,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998, entries=150, sequenceid=11, filesize=11.7 K 2024-12-10T01:15:31,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/bcce328365354897a5ca445d2ba8cdb0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0 2024-12-10T01:15:31,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0, entries=150, sequenceid=11, filesize=11.7 K 2024-12-10T01:15:31,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a435da89f77641c4b8c1dcc16a308ea9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9 2024-12-10T01:15:31,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9, entries=150, sequenceid=11, filesize=11.7 K 2024-12-10T01:15:31,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f61d17f4c75f3711af1be3dee84b7c70 in 595ms, sequenceid=11, compaction requested=false 2024-12-10T01:15:31,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:31,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-10T01:15:31,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6133c41bab314b338e097d41ca056845 is 50, key is test_row_0/A:col10/1733793330839/Put/seqid=0 2024-12-10T01:15:31,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742036_1212 (size=12001) 2024-12-10T01:15:31,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6133c41bab314b338e097d41ca056845 2024-12-10T01:15:31,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/4d01e566d12043c8ae02d86875459ac2 is 50, key is test_row_0/B:col10/1733793330839/Put/seqid=0 2024-12-10T01:15:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:31,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:31,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742037_1213 (size=12001) 2024-12-10T01:15:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793391459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793391462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793391479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793391489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793391489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793391569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793391569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793391779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793391779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:31,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/4d01e566d12043c8ae02d86875459ac2 2024-12-10T01:15:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/09c5979c614f4e4b8894356965778346 is 50, key is test_row_0/C:col10/1733793330839/Put/seqid=0 2024-12-10T01:15:31,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742038_1214 (size=12001) 2024-12-10T01:15:31,881 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/09c5979c614f4e4b8894356965778346 2024-12-10T01:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6133c41bab314b338e097d41ca056845 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845 2024-12-10T01:15:31,891 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T01:15:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/4d01e566d12043c8ae02d86875459ac2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2 2024-12-10T01:15:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:31,903 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T01:15:31,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/09c5979c614f4e4b8894356965778346 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346 2024-12-10T01:15:31,912 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T01:15:31,913 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f61d17f4c75f3711af1be3dee84b7c70 in 530ms, sequenceid=47, compaction requested=false 2024-12-10T01:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-10T01:15:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-10T01:15:31,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-10T01:15:31,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-12-10T01:15:31,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.1540 sec 2024-12-10T01:15:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:31,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:31,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4f9e465664b64215b8ff908a3c8622fd is 50, key is test_row_0/A:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742039_1215 (size=12001) 2024-12-10T01:15:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793392024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332015, max timestamp=1733793332027Num uniques 859; 2024-12-10T01:15:32,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332015, max timestamp=1733793332027Num uniques 859; 2024-12-10T01:15:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793392027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332015, max timestamp=1733793332027Num uniques 859; 2024-12-10T01:15:32,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793392093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793392095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793392095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793392139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793392139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793392201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793392209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793392209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793392350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793392350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793392410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4f9e465664b64215b8ff908a3c8622fd 2024-12-10T01:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793392419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793392419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b08263ab307344e18ad94910335149aa is 50, key is test_row_0/B:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742040_1216 (size=12001) 2024-12-10T01:15:32,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b08263ab307344e18ad94910335149aa 2024-12-10T01:15:32,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a8a9d0431a824b8c9bb1f1148a6df813 is 50, key is test_row_0/C:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742041_1217 (size=12001) 2024-12-10T01:15:32,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a8a9d0431a824b8c9bb1f1148a6df813 2024-12-10T01:15:32,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4f9e465664b64215b8ff908a3c8622fd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd 2024-12-10T01:15:32,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:15:32,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b08263ab307344e18ad94910335149aa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa 2024-12-10T01:15:32,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:15:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a8a9d0431a824b8c9bb1f1148a6df813 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813 2024-12-10T01:15:32,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:15:32,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f61d17f4c75f3711af1be3dee84b7c70 in 546ms, sequenceid=58, compaction requested=true 2024-12-10T01:15:32,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:32,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:32,538 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:32,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:32,538 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:32,540 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:32,540 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:32,540 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:32,540 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=35.2 K 2024-12-10T01:15:32,540 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:32,540 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting bcce328365354897a5ca445d2ba8cdb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733793330773 2024-12-10T01:15:32,540 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:32,541 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:32,541 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=35.2 K 2024-12-10T01:15:32,541 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d01e566d12043c8ae02d86875459ac2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733793330839 2024-12-10T01:15:32,541 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c57e9611cdd840fd8a11584243b00998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733793330773 2024-12-10T01:15:32,541 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b08263ab307344e18ad94910335149aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:32,542 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6133c41bab314b338e097d41ca056845, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733793330839 2024-12-10T01:15:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:32,543 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f9e465664b64215b8ff908a3c8622fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:32,554 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:32,555 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/07bd7b9af6ab4ea888295c9f3726390b is 50, key is test_row_0/A:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,555 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:32,556 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/105fe43dd6284e94a5797a283f341dbb is 50, key is test_row_0/B:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742042_1218 (size=12104) 2024-12-10T01:15:32,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742043_1219 (size=12104) 2024-12-10T01:15:32,588 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/105fe43dd6284e94a5797a283f341dbb as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/105fe43dd6284e94a5797a283f341dbb 2024-12-10T01:15:32,598 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 105fe43dd6284e94a5797a283f341dbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:32,598 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:32,598 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793332538; duration=0sec 2024-12-10T01:15:32,598 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:32,598 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:32,599 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:32,600 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:32,600 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:32,601 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:32,601 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=35.2 K 2024-12-10T01:15:32,601 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a435da89f77641c4b8c1dcc16a308ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733793330773 2024-12-10T01:15:32,602 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c5979c614f4e4b8894356965778346, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733793330839 2024-12-10T01:15:32,602 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a9d0431a824b8c9bb1f1148a6df813, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:32,609 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:32,614 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/ba45db1267774a84ab42d81b4876e787 is 50, key is test_row_0/C:col10/1733793331448/Put/seqid=0 2024-12-10T01:15:32,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742044_1220 (size=12104) 2024-12-10T01:15:32,622 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/ba45db1267774a84ab42d81b4876e787 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ba45db1267774a84ab42d81b4876e787 2024-12-10T01:15:32,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into ba45db1267774a84ab42d81b4876e787(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:32,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793332543; duration=0sec 2024-12-10T01:15:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:32,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793332091, max timestamp=1733793332662Num uniques 1018; 2024-12-10T01:15:32,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:32,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=105.11 KB heapSize=297.28 KB 2024-12-10T01:15:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793332091, max timestamp=1733793332662Num uniques 1018; 2024-12-10T01:15:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793332091, max timestamp=1733793332662Num uniques 1018; 2024-12-10T01:15:32,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:32,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/494247c14757415890130d4269b9124c is 50, key is test_row_0/A:col10/1733793332662/Put/seqid=0 2024-12-10T01:15:32,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742045_1221 (size=12001) 2024-12-10T01:15:32,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/494247c14757415890130d4269b9124c 2024-12-10T01:15:32,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/73e556675f064a81a2dd90ae07467afc is 50, key is test_row_0/B:col10/1733793332662/Put/seqid=0 2024-12-10T01:15:32,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332704, max timestamp=1733793332713Num uniques 859; 2024-12-10T01:15:32,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332704, max timestamp=1733793332713Num uniques 859; 2024-12-10T01:15:32,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793332704, max timestamp=1733793332713Num uniques 859; 2024-12-10T01:15:32,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742046_1222 (size=12001) 2024-12-10T01:15:32,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733793332740, max timestamp=1733793332752Num uniques 1018; 2024-12-10T01:15:32,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733793332740, max timestamp=1733793332752Num uniques 1018; 2024-12-10T01:15:32,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-10T01:15:32,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-10T01:15:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793332740, max timestamp=1733793332752Num uniques 1018; 2024-12-10T01:15:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:15:32,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793392782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793392782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733793332768, max timestamp=1733793332786Num uniques 1177; 2024-12-10T01:15:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793392783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793332768, max timestamp=1733793332786Num uniques 1177; 2024-12-10T01:15:32,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-10T01:15:32,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-10T01:15:32,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793332768, max timestamp=1733793332786Num uniques 1027; 2024-12-10T01:15:32,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793332815, max timestamp=1733793332827Num uniques 1086; 2024-12-10T01:15:32,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-10T01:15:32,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793332815, max timestamp=1733793332827Num uniques 1036; 2024-12-10T01:15:32,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T01:15:32,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793332815, max timestamp=1733793332827Num uniques 1036; 2024-12-10T01:15:32,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793332859, max timestamp=1733793332875Num uniques 1045; 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793332859, max timestamp=1733793332875Num uniques 1045; 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793332859, max timestamp=1733793332875Num uniques 1045; 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:15:32,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:15:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T01:15:32,910 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:32,910 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-10T01:15:32,911 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:32,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:32,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793392913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793392914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793392914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733793332911, max timestamp=1733793332917Num uniques 1104; 2024-12-10T01:15:32,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733793332911, max timestamp=1733793332917Num uniques 1104; 2024-12-10T01:15:32,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-10T01:15:32,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-10T01:15:32,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793392916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:32,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793332911, max timestamp=1733793332917Num uniques 1104; 2024-12-10T01:15:32,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-12-10T01:15:32,973 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/07bd7b9af6ab4ea888295c9f3726390b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/07bd7b9af6ab4ea888295c9f3726390b 2024-12-10T01:15:32,977 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into 07bd7b9af6ab4ea888295c9f3726390b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:32,977 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:32,977 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793332538; duration=0sec 2024-12-10T01:15:32,977 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:32,977 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733793332952, max timestamp=1733793332986Num uniques 1063; 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733793332952, max timestamp=1733793332986Num uniques 1063; 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733793332952, max timestamp=1733793332986Num uniques 1009; 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-10T01:15:32,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=7458, suffix heap size=132904,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=163 2024-12-10T01:15:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:33,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733793333030, max timestamp=1733793333035Num uniques 1022; 2024-12-10T01:15:33,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-10T01:15:33,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733793333030, max timestamp=1733793333035Num uniques 1022; 2024-12-10T01:15:33,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-10T01:15:33,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733793333030, max timestamp=1733793333035Num uniques 1022; 2024-12-10T01:15:33,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-10T01:15:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793393052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733793333050, max timestamp=1733793333054Num uniques 1131; 2024-12-10T01:15:33,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-12-10T01:15:33,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733793333050, max timestamp=1733793333054Num uniques 1131; 2024-12-10T01:15:33,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-12-10T01:15:33,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733793333050, max timestamp=1733793333054Num uniques 1131; 2024-12-10T01:15:33,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-12-10T01:15:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733793333069, max timestamp=1733793333078Num uniques 1040; 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733793333069, max timestamp=1733793333078Num uniques 1040; 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733793333069, max timestamp=1733793333078Num uniques 1040; 2024-12-10T01:15:33,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-10T01:15:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793393096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793393099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733793333093, max timestamp=1733793333100Num uniques 1099; 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733793333093, max timestamp=1733793333101Num uniques 1099; 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733793333093, max timestamp=1733793333101Num uniques 1099; 2024-12-10T01:15:33,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T01:15:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/73e556675f064a81a2dd90ae07467afc 2024-12-10T01:15:33,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/26df7e7c0aed4a0e8b4237f81e22e6b1 is 50, key is test_row_0/C:col10/1733793332662/Put/seqid=0 2024-12-10T01:15:33,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733793333137, max timestamp=1733793333146Num uniques 1058; 2024-12-10T01:15:33,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=9518, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-10T01:15:33,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733793333137, max timestamp=1733793333146Num uniques 1058; 2024-12-10T01:15:33,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=9518, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-10T01:15:33,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793333137, max timestamp=1733793333146Num uniques 1058; 2024-12-10T01:15:33,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=6870, suffix heap size=138000,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=150 2024-12-10T01:15:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742047_1223 (size=12001) 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733793333171, max timestamp=1733793333181Num uniques 1009; 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733793333171, max timestamp=1733793333181Num uniques 1067; 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733793333171, max timestamp=1733793333181Num uniques 1067; 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-10T01:15:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-12-10T01:15:33,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733793333205, max timestamp=1733793333213Num uniques 1026; 2024-12-10T01:15:33,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-10T01:15:33,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733793333205, max timestamp=1733793333213Num uniques 1026; 2024-12-10T01:15:33,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733793333205, max timestamp=1733793333213Num uniques 1026; 2024-12-10T01:15:33,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-10T01:15:33,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-10T01:15:33,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733793333229, max timestamp=1733793333236Num uniques 1085; 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733793333229, max timestamp=1733793333236Num uniques 1085; 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733793333229, max timestamp=1733793333236Num uniques 1085; 2024-12-10T01:15:33,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733793333253, max timestamp=1733793333263Num uniques 1044; 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733793333253, max timestamp=1733793333263Num uniques 1044; 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733793333253, max timestamp=1733793333263Num uniques 1044; 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-10T01:15:33,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-10T01:15:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793393281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793393282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793393284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733793333276, max timestamp=1733793333285Num uniques 1103; 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733793333276, max timestamp=1733793333285Num uniques 1103; 2024-12-10T01:15:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733793333276, max timestamp=1733793333285Num uniques 1103; 2024-12-10T01:15:33,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-10T01:15:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733793333305, max timestamp=1733793333325Num uniques 1062; 2024-12-10T01:15:33,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-10T01:15:33,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733793333305, max timestamp=1733793333325Num uniques 1062; 2024-12-10T01:15:33,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-10T01:15:33,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733793333305, max timestamp=1733793333325Num uniques 1062; 2024-12-10T01:15:33,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-10T01:15:33,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733793333325, max timestamp=1733793333362Num uniques 1021; 2024-12-10T01:15:33,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-10T01:15:33,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733793333325, max timestamp=1733793333362Num uniques 1021; 2024-12-10T01:15:33,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733793333325, max timestamp=1733793333362Num uniques 1021; 2024-12-10T01:15:33,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-10T01:15:33,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-10T01:15:33,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=330, cellSize=15110, totalHeapSize=34560, min timestamp=1733793333373, max timestamp=1733793333392Num uniques 1030; 2024-12-10T01:15:33,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=15110, suffix heap size=135088,new segment heap size=34560 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=330 2024-12-10T01:15:33,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733793333385, max timestamp=1733793333392Num uniques 1030; 2024-12-10T01:15:33,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-10T01:15:33,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793333385, max timestamp=1733793333392Num uniques 1030; 2024-12-10T01:15:33,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=6870, suffix heap size=135088,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=150 2024-12-10T01:15:33,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733793333402, max timestamp=1733793333414Num uniques 1039; 2024-12-10T01:15:33,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733793333402, max timestamp=1733793333414Num uniques 1189; 2024-12-10T01:15:33,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=13234, suffix heap size=136024,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=289 2024-12-10T01:15:33,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54454, new segment data size=13234, suffix heap size=151624,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1189, new segment cells count=289 2024-12-10T01:15:33,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733793333402, max timestamp=1733793333414Num uniques 1009; 2024-12-10T01:15:33,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13234, suffix heap size=132904,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=289 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793333425, max timestamp=1733793333433Num uniques 1148; 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793333425, max timestamp=1733793333433Num uniques 1148; 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733793333425, max timestamp=1733793333433Num uniques 1148; 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=6870, suffix heap size=147360,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=150 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=6870, suffix heap size=147360,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=150 2024-12-10T01:15:33,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=6870, suffix heap size=147360,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=150 2024-12-10T01:15:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733793333450, max timestamp=1733793333457Num uniques 1009; 2024-12-10T01:15:33,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7185, suffix heap size=132904,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=157 2024-12-10T01:15:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793393455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733793333450, max timestamp=1733793333457Num uniques 1009; 2024-12-10T01:15:33,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7185, suffix heap size=132904,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=157 2024-12-10T01:15:33,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1733793333450, max timestamp=1733793333457Num uniques 1009; 2024-12-10T01:15:33,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7185, suffix heap size=132904,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=157 2024-12-10T01:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733793333472, max timestamp=1733793333477Num uniques 1016; 2024-12-10T01:15:33,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733793333472, max timestamp=1733793333477Num uniques 1016; 2024-12-10T01:15:33,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733793333472, max timestamp=1733793333477Num uniques 1016; 2024-12-10T01:15:33,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=9886, suffix heap size=133632,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=216 2024-12-10T01:15:33,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=9886, suffix heap size=133632,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=216 2024-12-10T01:15:33,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=9886, suffix heap size=133632,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=216 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12590, totalHeapSize=28840, min timestamp=1733793333488, max timestamp=1733793333500Num uniques 1075; 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=12590, suffix heap size=139768,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=275 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12590, totalHeapSize=28840, min timestamp=1733793333488, max timestamp=1733793333500Num uniques 1075; 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=12590, suffix heap size=139768,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=275 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12590, totalHeapSize=28840, min timestamp=1733793333488, max timestamp=1733793333500Num uniques 1075; 2024-12-10T01:15:33,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=12590, suffix heap size=139768,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=275 2024-12-10T01:15:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793393516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1733793333512, max timestamp=1733793333518Num uniques 1134; 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=8424, suffix heap size=145904,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=184 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1733793333512, max timestamp=1733793333518Num uniques 1134; 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=8424, suffix heap size=145904,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=184 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1733793333512, max timestamp=1733793333518Num uniques 1134; 2024-12-10T01:15:33,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=8424, suffix heap size=145904,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=184 2024-12-10T01:15:33,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:33,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1733793333533, max timestamp=1733793333545Num uniques 1043; 2024-12-10T01:15:33,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=11128, suffix heap size=136440,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=243 2024-12-10T01:15:33,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1733793333533, max timestamp=1733793333545Num uniques 1043; 2024-12-10T01:15:33,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=11128, suffix heap size=136440,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=243 2024-12-10T01:15:33,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1733793333533, max timestamp=1733793333545Num uniques 1043; 2024-12-10T01:15:33,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=11128, suffix heap size=136440,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=243 2024-12-10T01:15:33,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/26df7e7c0aed4a0e8b4237f81e22e6b1 2024-12-10T01:15:33,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/494247c14757415890130d4269b9124c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c 2024-12-10T01:15:33,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:15:33,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/73e556675f064a81a2dd90ae07467afc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc 2024-12-10T01:15:33,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:15:33,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/26df7e7c0aed4a0e8b4237f81e22e6b1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1 2024-12-10T01:15:33,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:15:33,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=147.60 KB/151140 for f61d17f4c75f3711af1be3dee84b7c70 in 918ms, sequenceid=98, compaction requested=false 2024-12-10T01:15:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:33,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1733793333575, max timestamp=1733793333580Num uniques 1102; 2024-12-10T01:15:33,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50470, new segment data size=6960, suffix heap size=142576,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1102, new segment cells count=152 2024-12-10T01:15:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:33,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=111.82 KB heapSize=314.02 KB 2024-12-10T01:15:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:33,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1733793333575, max timestamp=1733793333580Num uniques 1102; 2024-12-10T01:15:33,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50470, new segment data size=6960, suffix heap size=142576,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1102, new segment cells count=152 2024-12-10T01:15:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:33,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6e22c7726ec2467386edba6b36829642 is 50, key is test_row_0/A:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742048_1224 (size=14741) 2024-12-10T01:15:33,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793393604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793393606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793393608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793393609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793393629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793393719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793393719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793393719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793393720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793393839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793393929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793393929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793393929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793393929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:33,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:33,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:33,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6e22c7726ec2467386edba6b36829642 2024-12-10T01:15:34,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/84b831e7c93a4046bf81fa61b8dbd3d3 is 50, key is test_row_0/B:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:34,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742049_1225 (size=12301) 2024-12-10T01:15:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:34,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:34,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793394149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/84b831e7c93a4046bf81fa61b8dbd3d3 2024-12-10T01:15:34,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/fa112621632d4671bb40904830a6799a is 50, key is test_row_0/C:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:34,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742050_1226 (size=12301) 2024-12-10T01:15:34,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793394659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,704 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:15:34,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793394749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793394749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793394749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793394750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:34,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/fa112621632d4671bb40904830a6799a 2024-12-10T01:15:34,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6e22c7726ec2467386edba6b36829642 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642 2024-12-10T01:15:34,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642, entries=200, sequenceid=584, filesize=14.4 K 2024-12-10T01:15:34,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/84b831e7c93a4046bf81fa61b8dbd3d3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3 2024-12-10T01:15:34,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3, entries=150, sequenceid=584, filesize=12.0 K 2024-12-10T01:15:34,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/fa112621632d4671bb40904830a6799a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a 2024-12-10T01:15:34,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a, entries=150, sequenceid=584, filesize=12.0 K 2024-12-10T01:15:34,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~190.41 KB/194976, currentSize=114.05 KB/116790 for f61d17f4c75f3711af1be3dee84b7c70 in 1276ms, sequenceid=584, compaction requested=true 2024-12-10T01:15:34,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:34,862 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:34,862 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:34,863 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:34,863 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:34,863 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,863 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/07bd7b9af6ab4ea888295c9f3726390b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=37.9 K 2024-12-10T01:15:34,863 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:34,863 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:34,863 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,863 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/105fe43dd6284e94a5797a283f341dbb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=35.6 K 2024-12-10T01:15:34,864 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bd7b9af6ab4ea888295c9f3726390b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:34,864 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 105fe43dd6284e94a5797a283f341dbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:34,865 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 494247c14757415890130d4269b9124c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793332093 2024-12-10T01:15:34,865 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e556675f064a81a2dd90ae07467afc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793332093 2024-12-10T01:15:34,865 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e22c7726ec2467386edba6b36829642, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:34,865 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b831e7c93a4046bf81fa61b8dbd3d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:34,886 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:34,886 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:34,887 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/164b5990c27f479fbfa03c4ad49b15e4 is 50, key is test_row_0/B:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:34,887 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/b7e25e127f434b95a6d42a3c9aa4bf6e is 50, key is test_row_0/A:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742051_1227 (size=12507) 2024-12-10T01:15:34,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742052_1228 (size=12507) 2024-12-10T01:15:34,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T01:15:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,900 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:34,901 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/b7e25e127f434b95a6d42a3c9aa4bf6e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/b7e25e127f434b95a6d42a3c9aa4bf6e 2024-12-10T01:15:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/39e4749f958744a0ae9b241ad4f39baf is 50, key is test_row_0/A:col10/1733793333598/Put/seqid=0 2024-12-10T01:15:34,912 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into b7e25e127f434b95a6d42a3c9aa4bf6e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:34,912 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:34,912 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793334861; duration=0sec 2024-12-10T01:15:34,912 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:34,912 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:34,912 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:34,914 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:34,914 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:34,914 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:34,914 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ba45db1267774a84ab42d81b4876e787, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=35.6 K 2024-12-10T01:15:34,915 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba45db1267774a84ab42d81b4876e787, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793331448 2024-12-10T01:15:34,915 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26df7e7c0aed4a0e8b4237f81e22e6b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793332093 2024-12-10T01:15:34,916 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa112621632d4671bb40904830a6799a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742053_1229 (size=12301) 2024-12-10T01:15:34,924 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:34,926 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/859d9fe7690044f99900bd3f30ac3a5d is 50, key is test_row_0/C:col10/1733793333578/Put/seqid=0 2024-12-10T01:15:34,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742054_1230 (size=12507) 2024-12-10T01:15:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:35,298 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/164b5990c27f479fbfa03c4ad49b15e4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/164b5990c27f479fbfa03c4ad49b15e4 2024-12-10T01:15:35,305 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 164b5990c27f479fbfa03c4ad49b15e4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:35,306 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:35,306 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793334862; duration=0sec 2024-12-10T01:15:35,306 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:35,306 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:35,323 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/39e4749f958744a0ae9b241ad4f39baf 2024-12-10T01:15:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/957e18e904a04f3a9660b1fbfd5d72f2 is 50, key is test_row_0/B:col10/1733793333598/Put/seqid=0 2024-12-10T01:15:35,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742055_1231 (size=12301) 2024-12-10T01:15:35,339 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/859d9fe7690044f99900bd3f30ac3a5d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/859d9fe7690044f99900bd3f30ac3a5d 2024-12-10T01:15:35,345 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 859d9fe7690044f99900bd3f30ac3a5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:35,345 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:35,345 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793334862; duration=0sec 2024-12-10T01:15:35,345 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:35,345 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:35,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793395705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:35,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/957e18e904a04f3a9660b1fbfd5d72f2 2024-12-10T01:15:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/72d7a6a49b8348b7937e82d792984384 is 50, key is test_row_0/C:col10/1733793333598/Put/seqid=0 2024-12-10T01:15:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742056_1232 (size=12301) 2024-12-10T01:15:35,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793395759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793395759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,761 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:35,761 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793395769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793395769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,771 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:35,772 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:35,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793395819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:35,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793396029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:36,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/72d7a6a49b8348b7937e82d792984384 2024-12-10T01:15:36,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/39e4749f958744a0ae9b241ad4f39baf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf 2024-12-10T01:15:36,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf, entries=150, sequenceid=604, filesize=12.0 K 2024-12-10T01:15:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/957e18e904a04f3a9660b1fbfd5d72f2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2 2024-12-10T01:15:36,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2, entries=150, sequenceid=604, filesize=12.0 K 2024-12-10T01:15:36,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/72d7a6a49b8348b7937e82d792984384 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384 2024-12-10T01:15:36,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384, entries=150, sequenceid=604, filesize=12.0 K 2024-12-10T01:15:36,165 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f61d17f4c75f3711af1be3dee84b7c70 in 1266ms, sequenceid=604, compaction requested=false 2024-12-10T01:15:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-10T01:15:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-10T01:15:36,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-10T01:15:36,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2540 sec 2024-12-10T01:15:36,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.2580 sec 2024-12-10T01:15:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:36,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:36,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:36,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/a2785a7189574a5888afb52afe2edabd is 50, key is test_row_0/A:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742057_1233 (size=12301) 2024-12-10T01:15:36,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/a2785a7189574a5888afb52afe2edabd 2024-12-10T01:15:36,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/799596c2f43e408ebe972ce10f020532 is 50, key is test_row_0/B:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742058_1234 (size=12301) 2024-12-10T01:15:36,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/799596c2f43e408ebe972ce10f020532 2024-12-10T01:15:36,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2c81efecee1e46c38dbf18eaf2f5c257 is 50, key is test_row_0/C:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742059_1235 (size=12301) 2024-12-10T01:15:36,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793396400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:36,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793396509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:36,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793396719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:36,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2c81efecee1e46c38dbf18eaf2f5c257 2024-12-10T01:15:36,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/a2785a7189574a5888afb52afe2edabd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd 2024-12-10T01:15:36,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd, entries=150, sequenceid=620, filesize=12.0 K 2024-12-10T01:15:36,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/799596c2f43e408ebe972ce10f020532 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532 2024-12-10T01:15:36,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532, entries=150, sequenceid=620, filesize=12.0 K 2024-12-10T01:15:36,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2c81efecee1e46c38dbf18eaf2f5c257 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257 2024-12-10T01:15:36,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257, entries=150, sequenceid=620, filesize=12.0 K 2024-12-10T01:15:36,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f61d17f4c75f3711af1be3dee84b7c70 in 459ms, sequenceid=620, compaction requested=true 2024-12-10T01:15:36,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:36,801 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:36,801 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:36,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:36,802 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:36,802 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:36,802 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:36,802 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/164b5990c27f479fbfa03c4ad49b15e4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.2 K 2024-12-10T01:15:36,803 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:36,803 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:36,803 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:36,803 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/b7e25e127f434b95a6d42a3c9aa4bf6e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.2 K 2024-12-10T01:15:36,804 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e25e127f434b95a6d42a3c9aa4bf6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:36,804 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 164b5990c27f479fbfa03c4ad49b15e4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:36,804 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 957e18e904a04f3a9660b1fbfd5d72f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733793333598 2024-12-10T01:15:36,804 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39e4749f958744a0ae9b241ad4f39baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733793333598 2024-12-10T01:15:36,806 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2785a7189574a5888afb52afe2edabd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:36,806 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 799596c2f43e408ebe972ce10f020532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:36,817 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:36,818 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/2a797da367cc45318c0ea00f9375fc3c is 50, key is test_row_0/A:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,819 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:36,819 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/c6153f0375414c51a45ed6d9f844cade is 50, key is test_row_0/B:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742060_1236 (size=12609) 2024-12-10T01:15:36,844 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/2a797da367cc45318c0ea00f9375fc3c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2a797da367cc45318c0ea00f9375fc3c 2024-12-10T01:15:36,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742061_1237 (size=12609) 2024-12-10T01:15:36,850 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into 2a797da367cc45318c0ea00f9375fc3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:36,850 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:36,850 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793336800; duration=0sec 2024-12-10T01:15:36,850 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:36,850 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:36,851 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:36,854 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:36,855 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:36,855 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:36,855 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/859d9fe7690044f99900bd3f30ac3a5d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.2 K 2024-12-10T01:15:36,855 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859d9fe7690044f99900bd3f30ac3a5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733793333575 2024-12-10T01:15:36,855 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/c6153f0375414c51a45ed6d9f844cade as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/c6153f0375414c51a45ed6d9f844cade 2024-12-10T01:15:36,855 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d7a6a49b8348b7937e82d792984384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733793333598 2024-12-10T01:15:36,856 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c81efecee1e46c38dbf18eaf2f5c257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:36,862 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into c6153f0375414c51a45ed6d9f844cade(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:36,862 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:36,862 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793336801; duration=0sec 2024-12-10T01:15:36,862 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:36,862 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:36,867 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:36,867 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/7f5f150c6a784be9805c6a43ba4d7853 is 50, key is test_row_0/C:col10/1733793335701/Put/seqid=0 2024-12-10T01:15:36,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742062_1238 (size=12609) 2024-12-10T01:15:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:37,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:37,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/2c4c508e4da34baeb10c98f0cda48912 is 50, key is test_row_0/A:col10/1733793336379/Put/seqid=0 2024-12-10T01:15:37,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742063_1239 (size=14741) 2024-12-10T01:15:37,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793397065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T01:15:37,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:37,069 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-10T01:15:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:37,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:37,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:37,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:37,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793397170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:37,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,290 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/7f5f150c6a784be9805c6a43ba4d7853 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7f5f150c6a784be9805c6a43ba4d7853 2024-12-10T01:15:37,294 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 7f5f150c6a784be9805c6a43ba4d7853(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:37,294 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:37,294 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793336801; duration=0sec 2024-12-10T01:15:37,294 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:37,294 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:37,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793397379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:37,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/2c4c508e4da34baeb10c98f0cda48912 2024-12-10T01:15:37,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/183341b502f747158d3278ed7d09deb0 is 50, key is test_row_0/B:col10/1733793336379/Put/seqid=0 2024-12-10T01:15:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742064_1240 (size=12301) 2024-12-10T01:15:37,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793397690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:37,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793397780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793397780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793397780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,782 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:37,782 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,783 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793397799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:37,802 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:37,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/183341b502f747158d3278ed7d09deb0 2024-12-10T01:15:37,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c9819facfc0f43228bfaec5786c8611d is 50, key is test_row_0/C:col10/1733793336379/Put/seqid=0 2024-12-10T01:15:37,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742065_1241 (size=12301) 2024-12-10T01:15:37,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:37,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:38,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793398200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:38,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c9819facfc0f43228bfaec5786c8611d 2024-12-10T01:15:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/2c4c508e4da34baeb10c98f0cda48912 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912 2024-12-10T01:15:38,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912, entries=200, sequenceid=642, filesize=14.4 K 2024-12-10T01:15:38,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:38,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/183341b502f747158d3278ed7d09deb0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0 2024-12-10T01:15:38,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0, entries=150, sequenceid=642, filesize=12.0 K 2024-12-10T01:15:38,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c9819facfc0f43228bfaec5786c8611d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d 2024-12-10T01:15:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d, entries=150, sequenceid=642, filesize=12.0 K 2024-12-10T01:15:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f61d17f4c75f3711af1be3dee84b7c70 in 1279ms, sequenceid=642, compaction requested=false 2024-12-10T01:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:38,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T01:15:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:38,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/e4673e0d5fc349b09f6c18aa3a559005 is 50, key is test_row_0/A:col10/1733793337053/Put/seqid=0 2024-12-10T01:15:38,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742066_1242 (size=12301) 2024-12-10T01:15:38,857 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/e4673e0d5fc349b09f6c18aa3a559005 2024-12-10T01:15:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/6b23af0fa3ef4c57bab91c86db0991df is 50, key is test_row_0/B:col10/1733793337053/Put/seqid=0 2024-12-10T01:15:38,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742067_1243 (size=12301) 2024-12-10T01:15:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:39,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793399271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:39,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/6b23af0fa3ef4c57bab91c86db0991df 2024-12-10T01:15:39,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/e7110dcb4a2046b695a7a918d60e9990 is 50, key is test_row_0/C:col10/1733793337053/Put/seqid=0 2024-12-10T01:15:39,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742068_1244 (size=12301) 2024-12-10T01:15:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793399379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793399589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:39,729 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/e7110dcb4a2046b695a7a918d60e9990 2024-12-10T01:15:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/e4673e0d5fc349b09f6c18aa3a559005 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005 2024-12-10T01:15:39,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005, entries=150, sequenceid=655, filesize=12.0 K 2024-12-10T01:15:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/6b23af0fa3ef4c57bab91c86db0991df as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df 2024-12-10T01:15:39,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df, entries=150, sequenceid=655, filesize=12.0 K 2024-12-10T01:15:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/e7110dcb4a2046b695a7a918d60e9990 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990 2024-12-10T01:15:39,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990, entries=150, sequenceid=655, filesize=12.0 K 2024-12-10T01:15:39,769 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f61d17f4c75f3711af1be3dee84b7c70 in 1320ms, sequenceid=655, compaction requested=true 2024-12-10T01:15:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-10T01:15:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-10T01:15:39,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-10T01:15:39,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7000 sec 2024-12-10T01:15:39,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.7040 sec 2024-12-10T01:15:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:39,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:39,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/5ae59d3bac3d4469a722a08c9970d511 is 50, key is test_row_0/A:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:39,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742069_1245 (size=14741) 2024-12-10T01:15:39,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793399943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:40,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793400059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793400269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:40,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/5ae59d3bac3d4469a722a08c9970d511 2024-12-10T01:15:40,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/23524367030b41fbbd10df3b6967568d is 50, key is test_row_0/B:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742070_1246 (size=12301) 2024-12-10T01:15:40,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/23524367030b41fbbd10df3b6967568d 2024-12-10T01:15:40,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6f2e83445c5748dc8613c815d2ebf156 is 50, key is test_row_0/C:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:40,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742071_1247 (size=12301) 2024-12-10T01:15:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793400579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:40,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6f2e83445c5748dc8613c815d2ebf156 2024-12-10T01:15:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/5ae59d3bac3d4469a722a08c9970d511 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511 2024-12-10T01:15:40,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511, entries=200, sequenceid=676, filesize=14.4 K 2024-12-10T01:15:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/23524367030b41fbbd10df3b6967568d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d 2024-12-10T01:15:40,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d, entries=150, sequenceid=676, filesize=12.0 K 2024-12-10T01:15:40,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6f2e83445c5748dc8613c815d2ebf156 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156 2024-12-10T01:15:40,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156, entries=150, sequenceid=676, filesize=12.0 K 2024-12-10T01:15:40,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f61d17f4c75f3711af1be3dee84b7c70 in 922ms, sequenceid=676, compaction requested=true 2024-12-10T01:15:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:40,824 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:40,824 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:40,825 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:40,825 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:40,825 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/c6153f0375414c51a45ed6d9f844cade, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=48.4 K 2024-12-10T01:15:40,825 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2a797da367cc45318c0ea00f9375fc3c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=53.1 K 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c6153f0375414c51a45ed6d9f844cade, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:40,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a797da367cc45318c0ea00f9375fc3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:40,826 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 183341b502f747158d3278ed7d09deb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733793336379 2024-12-10T01:15:40,826 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4c508e4da34baeb10c98f0cda48912, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733793336379 2024-12-10T01:15:40,826 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b23af0fa3ef4c57bab91c86db0991df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733793337053 2024-12-10T01:15:40,826 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 23524367030b41fbbd10df3b6967568d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:40,826 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4673e0d5fc349b09f6c18aa3a559005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733793337053 2024-12-10T01:15:40,827 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae59d3bac3d4469a722a08c9970d511, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:40,836 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:40,837 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/98f2d97879984d02927d32be45848a33 is 50, key is test_row_0/B:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:40,837 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:40,838 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/791f5c790c6b446bb24593104b863ad4 is 50, key is test_row_0/A:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742073_1249 (size=12745) 2024-12-10T01:15:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742072_1248 (size=12745) 2024-12-10T01:15:40,846 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/791f5c790c6b446bb24593104b863ad4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/791f5c790c6b446bb24593104b863ad4 2024-12-10T01:15:40,850 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into 791f5c790c6b446bb24593104b863ad4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:40,851 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:40,851 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=12, startTime=1733793340823; duration=0sec 2024-12-10T01:15:40,851 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:40,851 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:40,851 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:15:40,852 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:15:40,852 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:40,852 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:40,852 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7f5f150c6a784be9805c6a43ba4d7853, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=48.4 K 2024-12-10T01:15:40,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5f150c6a784be9805c6a43ba4d7853, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1733793335701 2024-12-10T01:15:40,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9819facfc0f43228bfaec5786c8611d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733793336379 2024-12-10T01:15:40,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7110dcb4a2046b695a7a918d60e9990, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1733793337053 2024-12-10T01:15:40,853 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2e83445c5748dc8613c815d2ebf156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:40,861 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:40,861 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/450080c2f7eb4a48885617e6c53eb911 is 50, key is test_row_0/C:col10/1733793339900/Put/seqid=0 2024-12-10T01:15:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742074_1250 (size=12745) 2024-12-10T01:15:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:41,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:41,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/75e78396906e4315b1f0b763273b773b is 50, key is test_row_0/A:col10/1733793341092/Put/seqid=0 2024-12-10T01:15:41,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742075_1251 (size=12301) 2024-12-10T01:15:41,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/75e78396906e4315b1f0b763273b773b 2024-12-10T01:15:41,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/9fd9d0c4ecb04bcd82e95a50ec5804dc is 50, key is test_row_0/B:col10/1733793341092/Put/seqid=0 2024-12-10T01:15:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742076_1252 (size=12301) 2024-12-10T01:15:41,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/9fd9d0c4ecb04bcd82e95a50ec5804dc 2024-12-10T01:15:41,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c3145c15d2ad4b39b1a7b782f5d9ec7a is 50, key is test_row_0/C:col10/1733793341092/Put/seqid=0 2024-12-10T01:15:41,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742077_1253 (size=12301) 2024-12-10T01:15:41,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c3145c15d2ad4b39b1a7b782f5d9ec7a 2024-12-10T01:15:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/75e78396906e4315b1f0b763273b773b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b 2024-12-10T01:15:41,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b, entries=150, sequenceid=689, filesize=12.0 K 2024-12-10T01:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/9fd9d0c4ecb04bcd82e95a50ec5804dc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc 2024-12-10T01:15:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793401175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc, entries=150, sequenceid=689, filesize=12.0 K 2024-12-10T01:15:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c3145c15d2ad4b39b1a7b782f5d9ec7a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a 2024-12-10T01:15:41,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a, entries=150, sequenceid=689, filesize=12.0 K 2024-12-10T01:15:41,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f61d17f4c75f3711af1be3dee84b7c70 in 101ms, sequenceid=689, compaction requested=false 2024-12-10T01:15:41,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T01:15:41,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:41,240 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-10T01:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:41,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:41,242 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:41,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:41,247 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/98f2d97879984d02927d32be45848a33 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/98f2d97879984d02927d32be45848a33 2024-12-10T01:15:41,251 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 98f2d97879984d02927d32be45848a33(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:41,251 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:41,251 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=12, startTime=1733793340824; duration=0sec 2024-12-10T01:15:41,252 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:41,252 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:41,270 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/450080c2f7eb4a48885617e6c53eb911 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/450080c2f7eb4a48885617e6c53eb911 2024-12-10T01:15:41,274 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 450080c2f7eb4a48885617e6c53eb911(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:41,274 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:41,274 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=12, startTime=1733793340824; duration=0sec 2024-12-10T01:15:41,274 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:41,274 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:41,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T01:15:41,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:41,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4df53ae2bad54e4da496c812a87f4dcf is 50, key is test_row_0/A:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:41,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742078_1254 (size=12301) 2024-12-10T01:15:41,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793401315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:41,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793401419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:41,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793401630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4df53ae2bad54e4da496c812a87f4dcf 2024-12-10T01:15:41,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/fc182c68dede40b088b6f10418d59db3 is 50, key is test_row_0/B:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:41,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742079_1255 (size=12301) 2024-12-10T01:15:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60662 deadline: 1733793401790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,795 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60718 deadline: 1733793401809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,810 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733793401809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,812 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733793401850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,852 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in f61d17f4c75f3711af1be3dee84b7c70 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:41,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:41,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:41,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:41,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:41,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793401940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:42,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:42,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/fc182c68dede40b088b6f10418d59db3 2024-12-10T01:15:42,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/ec92346213a94e90ab2a22b6ac06001f is 50, key is test_row_0/C:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:42,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742080_1256 (size=12301) 2024-12-10T01:15:42,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:42,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793402450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:42,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/ec92346213a94e90ab2a22b6ac06001f 2024-12-10T01:15:42,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/4df53ae2bad54e4da496c812a87f4dcf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf 2024-12-10T01:15:42,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf, entries=150, sequenceid=712, filesize=12.0 K 2024-12-10T01:15:42,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/fc182c68dede40b088b6f10418d59db3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3 2024-12-10T01:15:42,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3, entries=150, sequenceid=712, filesize=12.0 K 2024-12-10T01:15:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/ec92346213a94e90ab2a22b6ac06001f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f 2024-12-10T01:15:42,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f, entries=150, sequenceid=712, filesize=12.0 K 2024-12-10T01:15:42,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f61d17f4c75f3711af1be3dee84b7c70 in 1379ms, sequenceid=712, compaction requested=true 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:42,670 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:42,671 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:42,674 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:42,674 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:42,674 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:42,674 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,675 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/98f2d97879984d02927d32be45848a33, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.5 K 2024-12-10T01:15:42,675 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:42,675 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,675 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/791f5c790c6b446bb24593104b863ad4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.5 K 2024-12-10T01:15:42,676 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 791f5c790c6b446bb24593104b863ad4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:42,676 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f2d97879984d02927d32be45848a33, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:42,676 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd9d0c4ecb04bcd82e95a50ec5804dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733793339923 2024-12-10T01:15:42,676 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e78396906e4315b1f0b763273b773b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733793339923 2024-12-10T01:15:42,677 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc182c68dede40b088b6f10418d59db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:42,677 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4df53ae2bad54e4da496c812a87f4dcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:42,691 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:42,692 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/19600a5894354e55a1cb6d33060cfd4b is 50, key is test_row_0/A:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:42,703 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:42,703 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/a658c1c5b71242c69c301ddfab6bcb17 is 50, key is test_row_0/B:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:42,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742082_1258 (size=12847) 2024-12-10T01:15:42,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742081_1257 (size=12847) 2024-12-10T01:15:42,731 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/19600a5894354e55a1cb6d33060cfd4b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/19600a5894354e55a1cb6d33060cfd4b 2024-12-10T01:15:42,738 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into 19600a5894354e55a1cb6d33060cfd4b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:42,738 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:42,738 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793342670; duration=0sec 2024-12-10T01:15:42,738 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:42,738 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:42,738 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:42,739 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:42,739 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:42,739 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,739 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/450080c2f7eb4a48885617e6c53eb911, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.5 K 2024-12-10T01:15:42,740 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 450080c2f7eb4a48885617e6c53eb911, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1733793339261 2024-12-10T01:15:42,740 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c3145c15d2ad4b39b1a7b782f5d9ec7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733793339923 2024-12-10T01:15:42,741 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ec92346213a94e90ab2a22b6ac06001f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:42,748 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:42,749 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/14ebc64505a0430689ac59155ed32215 is 50, key is test_row_0/C:col10/1733793341159/Put/seqid=0 2024-12-10T01:15:42,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742083_1259 (size=12847) 2024-12-10T01:15:42,778 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/14ebc64505a0430689ac59155ed32215 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/14ebc64505a0430689ac59155ed32215 2024-12-10T01:15:42,788 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 14ebc64505a0430689ac59155ed32215(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:42,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:42,788 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793342670; duration=0sec 2024-12-10T01:15:42,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:42,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:42,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:42,795 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:42,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/60c7f47e0ef24131977b484f93c3cf91 is 50, key is test_row_0/A:col10/1733793341293/Put/seqid=0 2024-12-10T01:15:42,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742084_1260 (size=12301) 2024-12-10T01:15:43,124 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/a658c1c5b71242c69c301ddfab6bcb17 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/a658c1c5b71242c69c301ddfab6bcb17 2024-12-10T01:15:43,129 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into a658c1c5b71242c69c301ddfab6bcb17(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:43,129 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:43,129 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793342670; duration=0sec 2024-12-10T01:15:43,129 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:43,129 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:43,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/60c7f47e0ef24131977b484f93c3cf91 2024-12-10T01:15:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/360b33ab8a874ad0b363896ee178d665 is 50, key is test_row_0/B:col10/1733793341293/Put/seqid=0 2024-12-10T01:15:43,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742085_1261 (size=12301) 2024-12-10T01:15:43,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/360b33ab8a874ad0b363896ee178d665 2024-12-10T01:15:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2be04eb024e14efb836e5aa8346d035f is 50, key is test_row_0/C:col10/1733793341293/Put/seqid=0 2024-12-10T01:15:43,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742086_1262 (size=12301) 2024-12-10T01:15:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:43,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793343501, max timestamp=1733793343511Num uniques 859; 2024-12-10T01:15:43,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793343501, max timestamp=1733793343511Num uniques 859; 2024-12-10T01:15:43,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:43,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:43,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793343501, max timestamp=1733793343511Num uniques 859; 2024-12-10T01:15:43,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:43,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793403570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:43,644 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2be04eb024e14efb836e5aa8346d035f 2024-12-10T01:15:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/60c7f47e0ef24131977b484f93c3cf91 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91 2024-12-10T01:15:43,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91, entries=150, sequenceid=725, filesize=12.0 K 2024-12-10T01:15:43,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/360b33ab8a874ad0b363896ee178d665 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665 2024-12-10T01:15:43,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665, entries=150, sequenceid=725, filesize=12.0 K 2024-12-10T01:15:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/2be04eb024e14efb836e5aa8346d035f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f 2024-12-10T01:15:43,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f, entries=150, sequenceid=725, filesize=12.0 K 2024-12-10T01:15:43,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f61d17f4c75f3711af1be3dee84b7c70 in 872ms, sequenceid=725, compaction requested=false 2024-12-10T01:15:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-10T01:15:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-10T01:15:43,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-10T01:15:43,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4270 sec 2024-12-10T01:15:43,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.4310 sec 2024-12-10T01:15:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:43,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:43,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6ff71f396c0b45b084606c247228ee0c is 50, key is test_row_0/A:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742087_1263 (size=14741) 2024-12-10T01:15:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793403730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:43,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793403839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793404050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:44,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6ff71f396c0b45b084606c247228ee0c 2024-12-10T01:15:44,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b is 50, key is test_row_0/B:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742088_1264 (size=12301) 2024-12-10T01:15:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793404359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:44,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b 2024-12-10T01:15:44,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a64212bc27bb4cd59af21b0a10f9f4a8 is 50, key is test_row_0/C:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742089_1265 (size=12301) 2024-12-10T01:15:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793404870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:44,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a64212bc27bb4cd59af21b0a10f9f4a8 2024-12-10T01:15:44,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/6ff71f396c0b45b084606c247228ee0c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c 2024-12-10T01:15:44,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c, entries=200, sequenceid=763, filesize=14.4 K 2024-12-10T01:15:44,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b 2024-12-10T01:15:44,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b, entries=150, sequenceid=763, filesize=12.0 K 2024-12-10T01:15:44,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/a64212bc27bb4cd59af21b0a10f9f4a8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8 2024-12-10T01:15:44,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8, entries=150, sequenceid=763, filesize=12.0 K 2024-12-10T01:15:44,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f61d17f4c75f3711af1be3dee84b7c70 in 1286ms, sequenceid=763, compaction requested=true 2024-12-10T01:15:44,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:44,971 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:44,971 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:44,971 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:44,972 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/a658c1c5b71242c69c301ddfab6bcb17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.6 K 2024-12-10T01:15:44,972 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/19600a5894354e55a1cb6d33060cfd4b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=39.0 K 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a658c1c5b71242c69c301ddfab6bcb17, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19600a5894354e55a1cb6d33060cfd4b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 360b33ab8a874ad0b363896ee178d665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1733793341293 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c7f47e0ef24131977b484f93c3cf91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1733793341293 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6b0c2db8ce4b24b5c4b11eda4bdb3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:44,972 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff71f396c0b45b084606c247228ee0c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:44,980 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:44,980 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/1b433de3223f44fea2d5cefd42d4f272 is 50, key is test_row_0/A:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:44,983 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:44,983 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/36d6d3e197e64fdda152ef6e7a67085e is 50, key is test_row_0/B:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:44,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742091_1267 (size=12949) 2024-12-10T01:15:44,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742090_1266 (size=12949) 2024-12-10T01:15:45,004 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/36d6d3e197e64fdda152ef6e7a67085e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/36d6d3e197e64fdda152ef6e7a67085e 2024-12-10T01:15:45,009 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 36d6d3e197e64fdda152ef6e7a67085e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:45,009 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:45,009 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793344971; duration=0sec 2024-12-10T01:15:45,009 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:45,010 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:45,010 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:45,011 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:45,011 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:45,011 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:45,011 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/14ebc64505a0430689ac59155ed32215, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.6 K 2024-12-10T01:15:45,012 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ebc64505a0430689ac59155ed32215, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1733793341159 2024-12-10T01:15:45,012 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be04eb024e14efb836e5aa8346d035f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1733793341293 2024-12-10T01:15:45,012 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a64212bc27bb4cd59af21b0a10f9f4a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:45,018 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:45,019 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/9fd65c247143423daf293e90dee0b118 is 50, key is test_row_0/C:col10/1733793343682/Put/seqid=0 2024-12-10T01:15:45,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742092_1268 (size=12949) 2024-12-10T01:15:45,029 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/9fd65c247143423daf293e90dee0b118 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9fd65c247143423daf293e90dee0b118 2024-12-10T01:15:45,035 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 9fd65c247143423daf293e90dee0b118(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:45,035 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:45,035 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793344971; duration=0sec 2024-12-10T01:15:45,035 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:45,035 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T01:15:45,409 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:45,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-10T01:15:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T01:15:45,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:45,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:45,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:45,442 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/1b433de3223f44fea2d5cefd42d4f272 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/1b433de3223f44fea2d5cefd42d4f272 2024-12-10T01:15:45,449 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into 1b433de3223f44fea2d5cefd42d4f272(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:45,449 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:45,449 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793344970; duration=0sec 2024-12-10T01:15:45,449 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:45,449 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T01:15:45,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:45,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/9a05270a4c534f66b6c8558fc9792a97 is 50, key is test_row_0/A:col10/1733793343717/Put/seqid=0 2024-12-10T01:15:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742093_1269 (size=12301) 2024-12-10T01:15:45,604 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/9a05270a4c534f66b6c8558fc9792a97 2024-12-10T01:15:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/394c307e41d345a9ab6d99d9ddaad450 is 50, key is test_row_0/B:col10/1733793343717/Put/seqid=0 2024-12-10T01:15:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742094_1270 (size=12301) 2024-12-10T01:15:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T01:15:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793345935, max timestamp=1733793345945Num uniques 859; 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793345935, max timestamp=1733793345945Num uniques 859; 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793345935, max timestamp=1733793345945Num uniques 859; 2024-12-10T01:15:45,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:46,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/394c307e41d345a9ab6d99d9ddaad450 2024-12-10T01:15:46,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6d5a316445bf44cf9eaf4c4439c4759a is 50, key is test_row_0/C:col10/1733793343717/Put/seqid=0 2024-12-10T01:15:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742095_1271 (size=12301) 2024-12-10T01:15:46,024 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6d5a316445bf44cf9eaf4c4439c4759a 2024-12-10T01:15:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 371 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793406024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:46,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/9a05270a4c534f66b6c8558fc9792a97 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97 2024-12-10T01:15:46,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97, entries=150, sequenceid=776, filesize=12.0 K 2024-12-10T01:15:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/394c307e41d345a9ab6d99d9ddaad450 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450 2024-12-10T01:15:46,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450, entries=150, sequenceid=776, filesize=12.0 K 2024-12-10T01:15:46,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/6d5a316445bf44cf9eaf4c4439c4759a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a 2024-12-10T01:15:46,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a, entries=150, sequenceid=776, filesize=12.0 K 2024-12-10T01:15:46,040 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f61d17f4c75f3711af1be3dee84b7c70 in 446ms, sequenceid=776, compaction requested=false 2024-12-10T01:15:46,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:46,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-10T01:15:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-10T01:15:46,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-10T01:15:46,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 599 msec 2024-12-10T01:15:46,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 633 msec 2024-12-10T01:15:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T01:15:46,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:46,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-10T01:15:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:46,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:46,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:46,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:46,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:46,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/97926becbdd44fb494e1d7fc33190ea3 is 50, key is test_row_0/A:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:46,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742096_1272 (size=12301) 2024-12-10T01:15:46,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 379 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793406157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:46,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793406269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:46,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 381 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793406480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:46,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/97926becbdd44fb494e1d7fc33190ea3 2024-12-10T01:15:46,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/5be9ca6f498f455397871e17d289de35 is 50, key is test_row_0/B:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:46,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742097_1273 (size=12301) 2024-12-10T01:15:46,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793406789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:46,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/5be9ca6f498f455397871e17d289de35 2024-12-10T01:15:46,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/9cc4d1fb224a491cb5f997035ebc3a49 is 50, key is test_row_0/C:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:46,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742098_1274 (size=12301) 2024-12-10T01:15:46,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:46,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:47,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793407300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:47,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/9cc4d1fb224a491cb5f997035ebc3a49 2024-12-10T01:15:47,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/97926becbdd44fb494e1d7fc33190ea3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3 2024-12-10T01:15:47,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3, entries=150, sequenceid=814, filesize=12.0 K 2024-12-10T01:15:47,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/5be9ca6f498f455397871e17d289de35 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35 2024-12-10T01:15:47,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35, entries=150, sequenceid=814, filesize=12.0 K 2024-12-10T01:15:47,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/9cc4d1fb224a491cb5f997035ebc3a49 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49 2024-12-10T01:15:47,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49, entries=150, sequenceid=814, filesize=12.0 K 2024-12-10T01:15:47,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for f61d17f4c75f3711af1be3dee84b7c70 in 1258ms, sequenceid=814, compaction requested=true 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:47,390 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:47,390 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:47,392 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:47,392 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:47,392 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,392 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/1b433de3223f44fea2d5cefd42d4f272, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.7 K 2024-12-10T01:15:47,393 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:47,393 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:47,393 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,393 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/36d6d3e197e64fdda152ef6e7a67085e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.7 K 2024-12-10T01:15:47,393 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b433de3223f44fea2d5cefd42d4f272, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:47,393 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d6d3e197e64fdda152ef6e7a67085e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:47,393 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a05270a4c534f66b6c8558fc9792a97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1733793343693 2024-12-10T01:15:47,394 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 394c307e41d345a9ab6d99d9ddaad450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1733793343693 2024-12-10T01:15:47,394 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97926becbdd44fb494e1d7fc33190ea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:47,394 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be9ca6f498f455397871e17d289de35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:47,401 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:47,401 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/ecc0f7a70a9548918029e0210ceea6b4 is 50, key is test_row_0/A:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:47,402 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:47,403 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/6b2e484332094276ae9744bc6f1db8f2 is 50, key is test_row_0/B:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:47,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742099_1275 (size=13051) 2024-12-10T01:15:47,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742100_1276 (size=13051) 2024-12-10T01:15:47,420 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/ecc0f7a70a9548918029e0210ceea6b4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/ecc0f7a70a9548918029e0210ceea6b4 2024-12-10T01:15:47,427 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into ecc0f7a70a9548918029e0210ceea6b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:47,427 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:47,427 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793347390; duration=0sec 2024-12-10T01:15:47,427 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:47,427 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:47,428 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:47,428 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:47,429 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:47,429 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,429 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9fd65c247143423daf293e90dee0b118, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.7 K 2024-12-10T01:15:47,429 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd65c247143423daf293e90dee0b118, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1733793343522 2024-12-10T01:15:47,429 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5a316445bf44cf9eaf4c4439c4759a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1733793343693 2024-12-10T01:15:47,430 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc4d1fb224a491cb5f997035ebc3a49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:47,437 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:47,437 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/35eb29caa86948c0988478b6fd1de100 is 50, key is test_row_0/C:col10/1733793346009/Put/seqid=0 2024-12-10T01:15:47,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742101_1277 (size=13051) 2024-12-10T01:15:47,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:47,451 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/3076eebedd9140dca95ac2d711b1a2dc is 50, key is test_row_0/A:col10/1733793346145/Put/seqid=0 2024-12-10T01:15:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742102_1278 (size=12301) 2024-12-10T01:15:47,463 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/3076eebedd9140dca95ac2d711b1a2dc 2024-12-10T01:15:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/85a66391f02046cdbd67d43f46ed7483 is 50, key is test_row_0/B:col10/1733793346145/Put/seqid=0 2024-12-10T01:15:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742103_1279 (size=12301) 2024-12-10T01:15:47,818 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/6b2e484332094276ae9744bc6f1db8f2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b2e484332094276ae9744bc6f1db8f2 2024-12-10T01:15:47,822 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 6b2e484332094276ae9744bc6f1db8f2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:47,822 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:47,822 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793347390; duration=0sec 2024-12-10T01:15:47,822 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:47,822 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:47,847 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/35eb29caa86948c0988478b6fd1de100 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/35eb29caa86948c0988478b6fd1de100 2024-12-10T01:15:47,852 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into 35eb29caa86948c0988478b6fd1de100(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:47,852 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:47,852 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793347390; duration=0sec 2024-12-10T01:15:47,852 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:47,852 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:47,882 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/85a66391f02046cdbd67d43f46ed7483 2024-12-10T01:15:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/5116e87f022a429fb9c82e2e7c73fe49 is 50, key is test_row_0/C:col10/1733793346145/Put/seqid=0 2024-12-10T01:15:47,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742104_1280 (size=12301) 2024-12-10T01:15:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:48,296 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/5116e87f022a429fb9c82e2e7c73fe49 2024-12-10T01:15:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/3076eebedd9140dca95ac2d711b1a2dc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc 2024-12-10T01:15:48,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc, entries=150, sequenceid=824, filesize=12.0 K 2024-12-10T01:15:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/85a66391f02046cdbd67d43f46ed7483 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483 2024-12-10T01:15:48,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483, entries=150, sequenceid=824, filesize=12.0 K 2024-12-10T01:15:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/5116e87f022a429fb9c82e2e7c73fe49 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49 2024-12-10T01:15:48,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49, entries=150, sequenceid=824, filesize=12.0 K 2024-12-10T01:15:48,314 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for f61d17f4c75f3711af1be3dee84b7c70 in 863ms, sequenceid=824, compaction requested=false 2024-12-10T01:15:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-10T01:15:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-10T01:15:48,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-10T01:15:48,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-12-10T01:15:48,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2480 sec 2024-12-10T01:15:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:48,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:48,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/fae075998a2f43f5b3f344f42de84ff6 is 50, key is test_row_0/A:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:48,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742105_1281 (size=14741) 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793348380, max timestamp=1733793348397Num uniques 859; 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793348380, max timestamp=1733793348397Num uniques 859; 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793348380, max timestamp=1733793348397Num uniques 859; 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 425 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793408470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 426 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793408579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:48,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/fae075998a2f43f5b3f344f42de84ff6 2024-12-10T01:15:48,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/3ae39a9c17e74debae9f442ad02bfcd7 is 50, key is test_row_0/B:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:48,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742106_1282 (size=12301) 2024-12-10T01:15:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 427 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793408789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 428 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793409099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:49,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/3ae39a9c17e74debae9f442ad02bfcd7 2024-12-10T01:15:49,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/7bdec4a4a1aa43c58be83d1e244bd47e is 50, key is test_row_0/C:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:49,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742107_1283 (size=12301) 2024-12-10T01:15:49,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/7bdec4a4a1aa43c58be83d1e244bd47e 2024-12-10T01:15:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/fae075998a2f43f5b3f344f42de84ff6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6 2024-12-10T01:15:49,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6, entries=200, sequenceid=836, filesize=14.4 K 2024-12-10T01:15:49,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/3ae39a9c17e74debae9f442ad02bfcd7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7 2024-12-10T01:15:49,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7, entries=150, sequenceid=836, filesize=12.0 K 2024-12-10T01:15:49,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/7bdec4a4a1aa43c58be83d1e244bd47e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e 2024-12-10T01:15:49,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e, entries=150, sequenceid=836, filesize=12.0 K 2024-12-10T01:15:49,612 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T01:15:49,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f61d17f4c75f3711af1be3dee84b7c70 in 1275ms, sequenceid=836, compaction requested=true 2024-12-10T01:15:49,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:49,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:49,614 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:49,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:49,616 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:49,616 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:49,616 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/ecc0f7a70a9548918029e0210ceea6b4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=39.2 K 2024-12-10T01:15:49,616 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:49,616 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:49,616 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:49,616 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b2e484332094276ae9744bc6f1db8f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.8 K 2024-12-10T01:15:49,620 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2e484332094276ae9744bc6f1db8f2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:49,620 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc0f7a70a9548918029e0210ceea6b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:49,621 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3076eebedd9140dca95ac2d711b1a2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1733793346145 2024-12-10T01:15:49,621 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a66391f02046cdbd67d43f46ed7483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1733793346145 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793348459, max timestamp=1733793349616Num uniques 1018; 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793348459, max timestamp=1733793349616Num uniques 1018; 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793348459, max timestamp=1733793349616Num uniques 1018; 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:15:49,624 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae075998a2f43f5b3f344f42de84ff6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:49,624 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae39a9c17e74debae9f442ad02bfcd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:49,647 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:49,647 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/bb74f9fe4da94b59bfbd6c2fd7aca445 is 50, key is test_row_0/A:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:49,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-10T01:15:49,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:49,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:49,675 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:49,675 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/5bad0a133eee49eba9a4246cf8991f89 is 50, key is test_row_0/B:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:49,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/59e9a899512744358073adfdfbe66593 is 50, key is test_row_0/A:col10/1733793349640/Put/seqid=0 2024-12-10T01:15:49,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742108_1284 (size=13153) 2024-12-10T01:15:49,706 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/bb74f9fe4da94b59bfbd6c2fd7aca445 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/bb74f9fe4da94b59bfbd6c2fd7aca445 2024-12-10T01:15:49,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742109_1285 (size=13153) 2024-12-10T01:15:49,723 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into bb74f9fe4da94b59bfbd6c2fd7aca445(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:49,723 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:49,723 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793349613; duration=0sec 2024-12-10T01:15:49,723 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:49,723 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:49,724 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:49,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742110_1286 (size=12301) 2024-12-10T01:15:49,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=878 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/59e9a899512744358073adfdfbe66593 2024-12-10T01:15:49,725 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:49,725 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:49,725 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:49,726 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/35eb29caa86948c0988478b6fd1de100, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.8 K 2024-12-10T01:15:49,726 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35eb29caa86948c0988478b6fd1de100, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1733793346009 2024-12-10T01:15:49,727 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5116e87f022a429fb9c82e2e7c73fe49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1733793346145 2024-12-10T01:15:49,727 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bdec4a4a1aa43c58be83d1e244bd47e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:49,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ad26e394ac854497b1ff8398bcb98001 is 50, key is test_row_0/B:col10/1733793349640/Put/seqid=0 2024-12-10T01:15:49,732 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:49,732 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/b127a08283ff4015af86698a30b60aad is 50, key is test_row_0/C:col10/1733793348337/Put/seqid=0 2024-12-10T01:15:49,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742111_1287 (size=12301) 2024-12-10T01:15:49,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742112_1288 (size=13153) 2024-12-10T01:15:49,752 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/b127a08283ff4015af86698a30b60aad as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/b127a08283ff4015af86698a30b60aad 2024-12-10T01:15:49,759 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into b127a08283ff4015af86698a30b60aad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:49,759 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:49,759 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793349614; duration=0sec 2024-12-10T01:15:49,759 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:49,759 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793349730, max timestamp=1733793349785Num uniques 859; 2024-12-10T01:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793349730, max timestamp=1733793349785Num uniques 859; 2024-12-10T01:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793349730, max timestamp=1733793349785Num uniques 859; 2024-12-10T01:15:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:15:49,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 468 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793409893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 469 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793410010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:50,120 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/5bad0a133eee49eba9a4246cf8991f89 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5bad0a133eee49eba9a4246cf8991f89 2024-12-10T01:15:50,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=878 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ad26e394ac854497b1ff8398bcb98001 2024-12-10T01:15:50,144 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 5bad0a133eee49eba9a4246cf8991f89(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:50,144 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:50,144 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793349614; duration=0sec 2024-12-10T01:15:50,144 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:50,144 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:50,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/3fda581764324d518d0a5bfb58a334bf is 50, key is test_row_0/C:col10/1733793349640/Put/seqid=0 2024-12-10T01:15:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742113_1289 (size=12301) 2024-12-10T01:15:50,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=878 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/3fda581764324d518d0a5bfb58a334bf 2024-12-10T01:15:50,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/59e9a899512744358073adfdfbe66593 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593 2024-12-10T01:15:50,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593, entries=150, sequenceid=878, filesize=12.0 K 2024-12-10T01:15:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/ad26e394ac854497b1ff8398bcb98001 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001 2024-12-10T01:15:50,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001, entries=150, sequenceid=878, filesize=12.0 K 2024-12-10T01:15:50,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/3fda581764324d518d0a5bfb58a334bf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf 2024-12-10T01:15:50,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf, entries=150, sequenceid=878, filesize=12.0 K 2024-12-10T01:15:50,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for f61d17f4c75f3711af1be3dee84b7c70 in 569ms, sequenceid=878, compaction requested=false 2024-12-10T01:15:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:50,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:15:50,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/123dfc03bfea4b13930ee3e2ea9c349b is 50, key is test_row_0/A:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T01:15:50,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:50,230 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-10T01:15:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:50,232 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:50,233 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:50,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:50,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742114_1290 (size=12301) 2024-12-10T01:15:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 477 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793410255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 478 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793410369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:50,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T01:15:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T01:15:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 479 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60654 deadline: 1733793410579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:15:50,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/123dfc03bfea4b13930ee3e2ea9c349b 2024-12-10T01:15:50,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/e0a53cb12e71454194cb30e4407486d8 is 50, key is test_row_0/B:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:50,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742115_1291 (size=12301) 2024-12-10T01:15:50,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/e0a53cb12e71454194cb30e4407486d8 2024-12-10T01:15:50,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/75c6219893e2413d9fee6397a56a9b95 is 50, key is test_row_0/C:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:50,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T01:15:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:50,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742116_1292 (size=12301) 2024-12-10T01:15:50,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/75c6219893e2413d9fee6397a56a9b95 2024-12-10T01:15:50,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/123dfc03bfea4b13930ee3e2ea9c349b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b 2024-12-10T01:15:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b, entries=150, sequenceid=919, filesize=12.0 K 2024-12-10T01:15:50,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/e0a53cb12e71454194cb30e4407486d8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8 2024-12-10T01:15:50,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8, entries=150, sequenceid=919, filesize=12.0 K 2024-12-10T01:15:50,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/75c6219893e2413d9fee6397a56a9b95 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95 2024-12-10T01:15:50,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95, entries=150, sequenceid=919, filesize=12.0 K 2024-12-10T01:15:50,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for f61d17f4c75f3711af1be3dee84b7c70 in 550ms, sequenceid=919, compaction requested=true 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:15:50,772 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f61d17f4c75f3711af1be3dee84b7c70:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:15:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T01:15:50,773 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:50,775 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:50,775 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/B is initiating minor compaction (all files) 2024-12-10T01:15:50,775 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:50,775 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/A is initiating minor compaction (all files) 2024-12-10T01:15:50,775 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/B in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,775 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/A in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,775 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5bad0a133eee49eba9a4246cf8991f89, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.9 K 2024-12-10T01:15:50,775 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/bb74f9fe4da94b59bfbd6c2fd7aca445, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.9 K 2024-12-10T01:15:50,776 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb74f9fe4da94b59bfbd6c2fd7aca445, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:50,777 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e9a899512744358073adfdfbe66593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=878, earliestPutTs=1733793348468 2024-12-10T01:15:50,777 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bad0a133eee49eba9a4246cf8991f89, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:50,777 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123dfc03bfea4b13930ee3e2ea9c349b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733793349870 2024-12-10T01:15:50,777 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ad26e394ac854497b1ff8398bcb98001, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=878, earliestPutTs=1733793348468 2024-12-10T01:15:50,779 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a53cb12e71454194cb30e4407486d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733793349870 2024-12-10T01:15:50,781 INFO [Thread-1007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1007. 2024-12-10T01:15:50,782 DEBUG [Thread-1007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,782 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,782 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:50,782 INFO [Thread-1009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1009. 2024-12-10T01:15:50,782 DEBUG [Thread-1009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,782 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,783 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:50,783 INFO [Thread-1001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1001. 2024-12-10T01:15:50,783 DEBUG [Thread-1001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,783 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,783 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,784 INFO [Thread-1005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1005. 2024-12-10T01:15:50,784 DEBUG [Thread-1005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,784 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,784 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:50,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:50,788 INFO [Thread-1003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1003. 2024-12-10T01:15:50,788 DEBUG [Thread-1003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,788 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,788 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:50,794 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:50,794 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/f4583b44104d43fc9043eb2fdf65c2b1 is 50, key is test_row_0/A:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:50,796 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:50,796 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/410b2dacd77a4e25b20a12bfe5541ba1 is 50, key is test_row_0/B:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:50,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742117_1293 (size=13255) 2024-12-10T01:15:50,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742118_1294 (size=13255) 2024-12-10T01:15:50,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T01:15:50,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:50,847 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/053ca2868ca44964a2b1b7a7dd23b5d7 is 50, key is test_row_0/A:col10/1733793350246/Put/seqid=0 2024-12-10T01:15:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:50,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742119_1295 (size=12301) 2024-12-10T01:15:50,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/053ca2868ca44964a2b1b7a7dd23b5d7 2024-12-10T01:15:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b92421b1a2ba459187fde27dc96674c3 is 50, key is test_row_0/B:col10/1733793350246/Put/seqid=0 2024-12-10T01:15:50,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742120_1296 (size=12301) 2024-12-10T01:15:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. as already flushing 2024-12-10T01:15:50,891 INFO [Thread-996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-996. 2024-12-10T01:15:50,891 DEBUG [Thread-996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:50,891 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,891 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:50,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:51,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:15:51,208 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/f4583b44104d43fc9043eb2fdf65c2b1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/f4583b44104d43fc9043eb2fdf65c2b1 2024-12-10T01:15:51,208 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/410b2dacd77a4e25b20a12bfe5541ba1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/410b2dacd77a4e25b20a12bfe5541ba1 2024-12-10T01:15:51,212 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/B of f61d17f4c75f3711af1be3dee84b7c70 into 410b2dacd77a4e25b20a12bfe5541ba1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:51,212 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/A of f61d17f4c75f3711af1be3dee84b7c70 into f4583b44104d43fc9043eb2fdf65c2b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:51,212 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/B, priority=13, startTime=1733793350772; duration=0sec 2024-12-10T01:15:51,212 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/A, priority=13, startTime=1733793350772; duration=0sec 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:B 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:A 2024-12-10T01:15:51,212 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:15:51,213 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:15:51,213 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f61d17f4c75f3711af1be3dee84b7c70/C is initiating minor compaction (all files) 2024-12-10T01:15:51,213 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f61d17f4c75f3711af1be3dee84b7c70/C in TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:51,213 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/b127a08283ff4015af86698a30b60aad, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp, totalSize=36.9 K 2024-12-10T01:15:51,214 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b127a08283ff4015af86698a30b60aad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733793348326 2024-12-10T01:15:51,214 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fda581764324d518d0a5bfb58a334bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=878, earliestPutTs=1733793348468 2024-12-10T01:15:51,214 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c6219893e2413d9fee6397a56a9b95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733793349870 2024-12-10T01:15:51,220 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f61d17f4c75f3711af1be3dee84b7c70#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:15:51,220 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/cdbd66f67af4416187a57f11c96c2cd6 is 50, key is test_row_0/C:col10/1733793349870/Put/seqid=0 2024-12-10T01:15:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742121_1297 (size=13255) 2024-12-10T01:15:51,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b92421b1a2ba459187fde27dc96674c3 2024-12-10T01:15:51,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/67468bdc55cb438caeb70cb629cf7489 is 50, key is test_row_0/C:col10/1733793350246/Put/seqid=0 2024-12-10T01:15:51,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742122_1298 (size=12301) 2024-12-10T01:15:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:51,631 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/cdbd66f67af4416187a57f11c96c2cd6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/cdbd66f67af4416187a57f11c96c2cd6 2024-12-10T01:15:51,637 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f61d17f4c75f3711af1be3dee84b7c70/C of f61d17f4c75f3711af1be3dee84b7c70 into cdbd66f67af4416187a57f11c96c2cd6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:15:51,637 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:51,637 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., storeName=f61d17f4c75f3711af1be3dee84b7c70/C, priority=13, startTime=1733793350772; duration=0sec 2024-12-10T01:15:51,638 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:15:51,638 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f61d17f4c75f3711af1be3dee84b7c70:C 2024-12-10T01:15:51,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/67468bdc55cb438caeb70cb629cf7489 2024-12-10T01:15:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/053ca2868ca44964a2b1b7a7dd23b5d7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/053ca2868ca44964a2b1b7a7dd23b5d7 2024-12-10T01:15:51,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/053ca2868ca44964a2b1b7a7dd23b5d7, entries=150, sequenceid=928, filesize=12.0 K 2024-12-10T01:15:51,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/b92421b1a2ba459187fde27dc96674c3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b92421b1a2ba459187fde27dc96674c3 2024-12-10T01:15:51,712 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b92421b1a2ba459187fde27dc96674c3, entries=150, sequenceid=928, filesize=12.0 K 2024-12-10T01:15:51,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/67468bdc55cb438caeb70cb629cf7489 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/67468bdc55cb438caeb70cb629cf7489 2024-12-10T01:15:51,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/67468bdc55cb438caeb70cb629cf7489, entries=150, sequenceid=928, filesize=12.0 K 2024-12-10T01:15:51,717 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for f61d17f4c75f3711af1be3dee84b7c70 in 870ms, sequenceid=928, compaction requested=false 2024-12-10T01:15:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f61d17f4c75f3711af1be3dee84b7c70: 2024-12-10T01:15:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-10T01:15:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-10T01:15:51,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T01:15:51,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-12-10T01:15:51,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4910 sec 2024-12-10T01:15:51,853 INFO [Thread-994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-994. 2024-12-10T01:15:51,853 DEBUG [Thread-994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:51,853 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,853 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:51,891 INFO [Thread-990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-990. 2024-12-10T01:15:51,891 DEBUG [Thread-990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:51,891 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,891 DEBUG [Thread-990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:51,913 INFO [Thread-992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-992. 2024-12-10T01:15:51,913 DEBUG [Thread-992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:51,913 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,913 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:51,961 INFO [Thread-998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-998. 2024-12-10T01:15:51,961 DEBUG [Thread-998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:15:51,961 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,961 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:51,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T01:15:52,381 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 421 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4688 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4477 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4341 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4733 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4521 2024-12-10T01:15:52,381 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:15:52,381 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:15:52,382 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:15:52,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:52,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:52,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:15:52,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:15:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:52,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793352389"}]},"ts":"1733793352389"} 2024-12-10T01:15:52,390 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:15:52,390 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:15:52,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:15:52,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, UNASSIGN}] 2024-12-10T01:15:52,393 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, UNASSIGN 2024-12-10T01:15:52,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=f61d17f4c75f3711af1be3dee84b7c70, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:52,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, UNASSIGN because future has completed 2024-12-10T01:15:52,395 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:15:52,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:52,548 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(122): Close f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:52,548 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1722): Closing f61d17f4c75f3711af1be3dee84b7c70, disabling compactions & flushes 2024-12-10T01:15:52,549 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. after waiting 0 ms 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:52,549 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(2902): Flushing f61d17f4c75f3711af1be3dee84b7c70 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=A 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=B 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f61d17f4c75f3711af1be3dee84b7c70, store=C 2024-12-10T01:15:52,549 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:52,553 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/85daa7754b264ffba78dca27bf5315f8 is 50, key is test_row_0/A:col10/1733793351910/Put/seqid=0 2024-12-10T01:15:52,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742123_1299 (size=12301) 2024-12-10T01:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:52,958 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/85daa7754b264ffba78dca27bf5315f8 2024-12-10T01:15:52,965 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/226b4efcd98a48009339ddd83f360970 is 50, key is test_row_0/B:col10/1733793351910/Put/seqid=0 2024-12-10T01:15:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:53,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742124_1300 (size=12301) 2024-12-10T01:15:53,522 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/226b4efcd98a48009339ddd83f360970 2024-12-10T01:15:53,528 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c2a603d34d0542f3a347e662d8459cf9 is 50, key is test_row_0/C:col10/1733793351910/Put/seqid=0 2024-12-10T01:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:53,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742125_1301 (size=12301) 2024-12-10T01:15:53,932 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c2a603d34d0542f3a347e662d8459cf9 2024-12-10T01:15:53,938 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/A/85daa7754b264ffba78dca27bf5315f8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/85daa7754b264ffba78dca27bf5315f8 2024-12-10T01:15:53,944 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/85daa7754b264ffba78dca27bf5315f8, entries=150, sequenceid=939, filesize=12.0 K 2024-12-10T01:15:53,945 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/B/226b4efcd98a48009339ddd83f360970 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/226b4efcd98a48009339ddd83f360970 2024-12-10T01:15:53,948 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/226b4efcd98a48009339ddd83f360970, entries=150, sequenceid=939, filesize=12.0 K 2024-12-10T01:15:53,949 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/.tmp/C/c2a603d34d0542f3a347e662d8459cf9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c2a603d34d0542f3a347e662d8459cf9 2024-12-10T01:15:53,952 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c2a603d34d0542f3a347e662d8459cf9, entries=150, sequenceid=939, filesize=12.0 K 2024-12-10T01:15:53,953 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f61d17f4c75f3711af1be3dee84b7c70 in 1404ms, sequenceid=939, compaction requested=true 2024-12-10T01:15:53,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/07bd7b9af6ab4ea888295c9f3726390b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/b7e25e127f434b95a6d42a3c9aa4bf6e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2a797da367cc45318c0ea00f9375fc3c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/791f5c790c6b446bb24593104b863ad4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/19600a5894354e55a1cb6d33060cfd4b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/1b433de3223f44fea2d5cefd42d4f272, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/ecc0f7a70a9548918029e0210ceea6b4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/bb74f9fe4da94b59bfbd6c2fd7aca445, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b] to archive 2024-12-10T01:15:53,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6133c41bab314b338e097d41ca056845 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4f9e465664b64215b8ff908a3c8622fd 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/494247c14757415890130d4269b9124c 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/07bd7b9af6ab4ea888295c9f3726390b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/07bd7b9af6ab4ea888295c9f3726390b 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/39e4749f958744a0ae9b241ad4f39baf 2024-12-10T01:15:53,956 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6e22c7726ec2467386edba6b36829642 2024-12-10T01:15:53,957 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/b7e25e127f434b95a6d42a3c9aa4bf6e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/b7e25e127f434b95a6d42a3c9aa4bf6e 2024-12-10T01:15:53,957 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/c57e9611cdd840fd8a11584243b00998 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/a2785a7189574a5888afb52afe2edabd 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2a797da367cc45318c0ea00f9375fc3c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2a797da367cc45318c0ea00f9375fc3c 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/791f5c790c6b446bb24593104b863ad4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/791f5c790c6b446bb24593104b863ad4 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/5ae59d3bac3d4469a722a08c9970d511 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/e4673e0d5fc349b09f6c18aa3a559005 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/2c4c508e4da34baeb10c98f0cda48912 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/75e78396906e4315b1f0b763273b773b 2024-12-10T01:15:53,958 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/19600a5894354e55a1cb6d33060cfd4b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/19600a5894354e55a1cb6d33060cfd4b 2024-12-10T01:15:53,959 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/4df53ae2bad54e4da496c812a87f4dcf 2024-12-10T01:15:53,959 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/9a05270a4c534f66b6c8558fc9792a97 2024-12-10T01:15:53,959 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/1b433de3223f44fea2d5cefd42d4f272 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/1b433de3223f44fea2d5cefd42d4f272 2024-12-10T01:15:53,959 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/60c7f47e0ef24131977b484f93c3cf91 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/97926becbdd44fb494e1d7fc33190ea3 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/ecc0f7a70a9548918029e0210ceea6b4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/ecc0f7a70a9548918029e0210ceea6b4 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/3076eebedd9140dca95ac2d711b1a2dc 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/6ff71f396c0b45b084606c247228ee0c 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/59e9a899512744358073adfdfbe66593 2024-12-10T01:15:53,960 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/123dfc03bfea4b13930ee3e2ea9c349b 2024-12-10T01:15:53,961 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/fae075998a2f43f5b3f344f42de84ff6 2024-12-10T01:15:53,961 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/bb74f9fe4da94b59bfbd6c2fd7aca445 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/bb74f9fe4da94b59bfbd6c2fd7aca445 2024-12-10T01:15:53,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/105fe43dd6284e94a5797a283f341dbb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/164b5990c27f479fbfa03c4ad49b15e4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/c6153f0375414c51a45ed6d9f844cade, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/98f2d97879984d02927d32be45848a33, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/a658c1c5b71242c69c301ddfab6bcb17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/36d6d3e197e64fdda152ef6e7a67085e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b2e484332094276ae9744bc6f1db8f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5bad0a133eee49eba9a4246cf8991f89, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8] to archive 2024-12-10T01:15:53,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:53,963 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/bcce328365354897a5ca445d2ba8cdb0 2024-12-10T01:15:53,964 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/4d01e566d12043c8ae02d86875459ac2 2024-12-10T01:15:53,964 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/105fe43dd6284e94a5797a283f341dbb to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/105fe43dd6284e94a5797a283f341dbb 2024-12-10T01:15:53,964 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/164b5990c27f479fbfa03c4ad49b15e4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/164b5990c27f479fbfa03c4ad49b15e4 2024-12-10T01:15:53,964 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/84b831e7c93a4046bf81fa61b8dbd3d3 2024-12-10T01:15:53,964 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/957e18e904a04f3a9660b1fbfd5d72f2 2024-12-10T01:15:53,965 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/c6153f0375414c51a45ed6d9f844cade to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/c6153f0375414c51a45ed6d9f844cade 2024-12-10T01:15:53,965 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/799596c2f43e408ebe972ce10f020532 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b23af0fa3ef4c57bab91c86db0991df 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b08263ab307344e18ad94910335149aa 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/183341b502f747158d3278ed7d09deb0 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/23524367030b41fbbd10df3b6967568d 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/98f2d97879984d02927d32be45848a33 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/98f2d97879984d02927d32be45848a33 2024-12-10T01:15:53,966 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/73e556675f064a81a2dd90ae07467afc 2024-12-10T01:15:53,967 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/9fd9d0c4ecb04bcd82e95a50ec5804dc 2024-12-10T01:15:53,967 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/fc182c68dede40b088b6f10418d59db3 2024-12-10T01:15:53,967 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/a658c1c5b71242c69c301ddfab6bcb17 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/a658c1c5b71242c69c301ddfab6bcb17 2024-12-10T01:15:53,967 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/36d6d3e197e64fdda152ef6e7a67085e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/36d6d3e197e64fdda152ef6e7a67085e 2024-12-10T01:15:53,968 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/360b33ab8a874ad0b363896ee178d665 2024-12-10T01:15:53,968 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ed6b0c2db8ce4b24b5c4b11eda4bdb3b 2024-12-10T01:15:53,968 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b2e484332094276ae9744bc6f1db8f2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/6b2e484332094276ae9744bc6f1db8f2 2024-12-10T01:15:53,968 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/394c307e41d345a9ab6d99d9ddaad450 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5be9ca6f498f455397871e17d289de35 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/85a66391f02046cdbd67d43f46ed7483 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5bad0a133eee49eba9a4246cf8991f89 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/5bad0a133eee49eba9a4246cf8991f89 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/3ae39a9c17e74debae9f442ad02bfcd7 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/ad26e394ac854497b1ff8398bcb98001 2024-12-10T01:15:53,969 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/e0a53cb12e71454194cb30e4407486d8 2024-12-10T01:15:53,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ba45db1267774a84ab42d81b4876e787, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/859d9fe7690044f99900bd3f30ac3a5d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7f5f150c6a784be9805c6a43ba4d7853, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/450080c2f7eb4a48885617e6c53eb911, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/14ebc64505a0430689ac59155ed32215, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9fd65c247143423daf293e90dee0b118, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/35eb29caa86948c0988478b6fd1de100, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/b127a08283ff4015af86698a30b60aad, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95] to archive 2024-12-10T01:15:53,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:15:53,972 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ba45db1267774a84ab42d81b4876e787 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ba45db1267774a84ab42d81b4876e787 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a435da89f77641c4b8c1dcc16a308ea9 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a8a9d0431a824b8c9bb1f1148a6df813 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/fa112621632d4671bb40904830a6799a 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/26df7e7c0aed4a0e8b4237f81e22e6b1 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/72d7a6a49b8348b7937e82d792984384 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/09c5979c614f4e4b8894356965778346 2024-12-10T01:15:53,973 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/859d9fe7690044f99900bd3f30ac3a5d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/859d9fe7690044f99900bd3f30ac3a5d 2024-12-10T01:15:53,974 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7f5f150c6a784be9805c6a43ba4d7853 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7f5f150c6a784be9805c6a43ba4d7853 2024-12-10T01:15:53,974 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2c81efecee1e46c38dbf18eaf2f5c257 2024-12-10T01:15:53,974 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c9819facfc0f43228bfaec5786c8611d 2024-12-10T01:15:53,975 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6f2e83445c5748dc8613c815d2ebf156 2024-12-10T01:15:53,975 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/450080c2f7eb4a48885617e6c53eb911 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/450080c2f7eb4a48885617e6c53eb911 2024-12-10T01:15:53,975 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/e7110dcb4a2046b695a7a918d60e9990 2024-12-10T01:15:53,975 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/14ebc64505a0430689ac59155ed32215 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/14ebc64505a0430689ac59155ed32215 2024-12-10T01:15:53,975 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c3145c15d2ad4b39b1a7b782f5d9ec7a 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/2be04eb024e14efb836e5aa8346d035f 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/ec92346213a94e90ab2a22b6ac06001f 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/6d5a316445bf44cf9eaf4c4439c4759a 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/a64212bc27bb4cd59af21b0a10f9f4a8 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9fd65c247143423daf293e90dee0b118 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9fd65c247143423daf293e90dee0b118 2024-12-10T01:15:53,977 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/5116e87f022a429fb9c82e2e7c73fe49 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/35eb29caa86948c0988478b6fd1de100 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/35eb29caa86948c0988478b6fd1de100 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/9cc4d1fb224a491cb5f997035ebc3a49 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/7bdec4a4a1aa43c58be83d1e244bd47e 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/3fda581764324d518d0a5bfb58a334bf 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/75c6219893e2413d9fee6397a56a9b95 2024-12-10T01:15:53,978 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/b127a08283ff4015af86698a30b60aad to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/b127a08283ff4015af86698a30b60aad 2024-12-10T01:15:53,982 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/recovered.edits/942.seqid, newMaxSeqId=942, maxSeqId=1 2024-12-10T01:15:53,982 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. 2024-12-10T01:15:53,982 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1676): Region close journal for f61d17f4c75f3711af1be3dee84b7c70: Waiting for close lock at 1733793352549Running coprocessor pre-close hooks at 1733793352549Disabling compacts and flushes for region at 1733793352549Disabling writes for close at 1733793352549Obtaining lock to block concurrent updates at 1733793352549Preparing flush snapshotting stores in f61d17f4c75f3711af1be3dee84b7c70 at 1733793352549Finished memstore snapshotting TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733793352549Flushing stores of TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70. at 1733793352550 (+1 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/A: creating writer at 1733793352550Flushing f61d17f4c75f3711af1be3dee84b7c70/A: appending metadata at 1733793352553 (+3 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/A: closing flushed file at 1733793352553Flushing f61d17f4c75f3711af1be3dee84b7c70/B: creating writer at 1733793352962 (+409 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/B: appending metadata at 1733793352965 (+3 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/B: closing flushed file at 1733793352965Flushing f61d17f4c75f3711af1be3dee84b7c70/C: creating writer at 1733793353526 (+561 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/C: appending metadata at 1733793353528 (+2 ms)Flushing f61d17f4c75f3711af1be3dee84b7c70/C: closing flushed file at 1733793353528Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cbd64c4: reopening flushed file at 1733793353937 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5318f0c8: reopening flushed file at 1733793353944 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cffc0b2: reopening flushed file at 1733793353948 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f61d17f4c75f3711af1be3dee84b7c70 in 1404ms, sequenceid=939, compaction requested=true at 1733793353953 (+5 ms)Writing region close event to WAL at 1733793353979 (+26 ms)Running coprocessor post-close hooks at 1733793353982 (+3 ms)Closed at 1733793353982 2024-12-10T01:15:53,984 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(157): Closed f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:53,984 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=f61d17f4c75f3711af1be3dee84b7c70, regionState=CLOSED 2024-12-10T01:15:53,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:53,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T01:15:53,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure f61d17f4c75f3711af1be3dee84b7c70, server=fd1a2b66caf8,38925,1733793262612 in 1.5910 sec 2024-12-10T01:15:53,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-12-10T01:15:53,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f61d17f4c75f3711af1be3dee84b7c70, UNASSIGN in 1.5960 sec 2024-12-10T01:15:53,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-10T01:15:53,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5980 sec 2024-12-10T01:15:53,992 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793353992"}]},"ts":"1733793353992"} 2024-12-10T01:15:53,993 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:15:53,993 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:15:53,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6110 sec 2024-12-10T01:15:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T01:15:54,540 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:54,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:15:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,544 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T01:15:54,546 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=74, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,548 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:54,552 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/recovered.edits] 2024-12-10T01:15:54,557 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/85daa7754b264ffba78dca27bf5315f8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/85daa7754b264ffba78dca27bf5315f8 2024-12-10T01:15:54,558 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/053ca2868ca44964a2b1b7a7dd23b5d7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/053ca2868ca44964a2b1b7a7dd23b5d7 2024-12-10T01:15:54,558 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/f4583b44104d43fc9043eb2fdf65c2b1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/A/f4583b44104d43fc9043eb2fdf65c2b1 2024-12-10T01:15:54,562 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/226b4efcd98a48009339ddd83f360970 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/226b4efcd98a48009339ddd83f360970 2024-12-10T01:15:54,562 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/410b2dacd77a4e25b20a12bfe5541ba1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/410b2dacd77a4e25b20a12bfe5541ba1 2024-12-10T01:15:54,562 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b92421b1a2ba459187fde27dc96674c3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/B/b92421b1a2ba459187fde27dc96674c3 2024-12-10T01:15:54,566 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/67468bdc55cb438caeb70cb629cf7489 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/67468bdc55cb438caeb70cb629cf7489 2024-12-10T01:15:54,566 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c2a603d34d0542f3a347e662d8459cf9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/c2a603d34d0542f3a347e662d8459cf9 2024-12-10T01:15:54,566 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/cdbd66f67af4416187a57f11c96c2cd6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/C/cdbd66f67af4416187a57f11c96c2cd6 2024-12-10T01:15:54,569 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/recovered.edits/942.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70/recovered.edits/942.seqid 2024-12-10T01:15:54,570 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f61d17f4c75f3711af1be3dee84b7c70 2024-12-10T01:15:54,570 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:15:54,572 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=74, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,575 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:15:54,576 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:15:54,577 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=74, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,578 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:15:54,578 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793354578"}]},"ts":"9223372036854775807"} 2024-12-10T01:15:54,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:15:54,579 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f61d17f4c75f3711af1be3dee84b7c70, NAME => 'TestAcidGuarantees,,1733793329582.f61d17f4c75f3711af1be3dee84b7c70.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:15:54,579 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:15:54,580 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793354579"}]},"ts":"9223372036854775807"} 2024-12-10T01:15:54,581 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:15:54,582 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=74, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-10T01:15:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T01:15:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:15:54,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:54,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=256 (was 255) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 368) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7573 (was 8186) 2024-12-10T01:15:54,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=256, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=7573 2024-12-10T01:15:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:15:54,670 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:15:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:54,672 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:15:54,672 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:54,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 75 2024-12-10T01:15:54,674 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:15:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742126_1302 (size=1061) 2024-12-10T01:15:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:55,081 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:15:55,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742127_1303 (size=53) 2024-12-10T01:15:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 301a23263bb8ba2547c849aa87975635, disabling compactions & flushes 2024-12-10T01:15:55,488 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. after waiting 0 ms 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,488 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,488 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 301a23263bb8ba2547c849aa87975635: Waiting for close lock at 1733793355488Disabling compacts and flushes for region at 1733793355488Disabling writes for close at 1733793355488Writing region close event to WAL at 1733793355488Closed at 1733793355488 2024-12-10T01:15:55,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:15:55,491 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793355490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793355490"}]},"ts":"1733793355490"} 2024-12-10T01:15:55,493 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:15:55,495 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:15:55,495 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793355495"}]},"ts":"1733793355495"} 2024-12-10T01:15:55,496 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:15:55,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, ASSIGN}] 2024-12-10T01:15:55,498 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, ASSIGN 2024-12-10T01:15:55,499 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:15:55,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:55,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, ASSIGN because future has completed 2024-12-10T01:15:55,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:55,812 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,813 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:15:55,813 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,813 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:15:55,813 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,814 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,816 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,817 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:55,818 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName A 2024-12-10T01:15:55,818 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:55,818 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:55,819 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,820 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:55,820 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName B 2024-12-10T01:15:55,820 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:55,821 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:55,821 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,822 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:55,822 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName C 2024-12-10T01:15:55,823 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:55,823 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:55,823 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,823 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,824 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,825 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,826 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,826 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,827 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:15:55,828 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,830 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:15:55,830 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 301a23263bb8ba2547c849aa87975635; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67295149, jitterRate=0.0027758628129959106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:15:55,831 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:55,831 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 301a23263bb8ba2547c849aa87975635: Running coprocessor pre-open hook at 1733793355814Writing region info on filesystem at 1733793355814Initializing all the Stores at 1733793355815 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793355815Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793355815Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793355815Cleaning up temporary data from old regions at 1733793355826 (+11 ms)Running coprocessor post-open hooks at 1733793355831 (+5 ms)Region opened successfully at 1733793355831 2024-12-10T01:15:55,832 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., pid=77, masterSystemTime=1733793355807 2024-12-10T01:15:55,834 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,834 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:55,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:55,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:55,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-10T01:15:55,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 in 183 msec 2024-12-10T01:15:55,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-10T01:15:55,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, ASSIGN in 342 msec 2024-12-10T01:15:55,843 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:15:55,843 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793355843"}]},"ts":"1733793355843"} 2024-12-10T01:15:55,845 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:15:55,845 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:15:55,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-10T01:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-10T01:15:56,820 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:56,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eba5433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:56,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:56,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:56,825 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e8aa1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:56,829 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:56,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665842f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:56,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:56,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:56,832 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:15:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:15:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56114d57 2024-12-10T01:15:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:15:56,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:15:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:15:56,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T01:15:56,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742128_1304 (size=1097) 2024-12-10T01:15:57,252 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T01:15:57,252 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T01:15:57,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:15:57,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE}] 2024-12-10T01:15:57,261 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE 2024-12-10T01:15:57,262 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE because future has completed 2024-12-10T01:15:57,264 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:15:57,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:57,418 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,418 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:15:57,418 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 301a23263bb8ba2547c849aa87975635, disabling compactions & flushes 2024-12-10T01:15:57,418 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,418 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,418 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. after waiting 0 ms 2024-12-10T01:15:57,418 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,472 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T01:15:57,473 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,473 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 301a23263bb8ba2547c849aa87975635: Waiting for close lock at 1733793357418Running coprocessor pre-close hooks at 1733793357418Disabling compacts and flushes for region at 1733793357418Disabling writes for close at 1733793357418Writing region close event to WAL at 1733793357418Running coprocessor post-close hooks at 1733793357473 (+55 ms)Closed at 1733793357473 2024-12-10T01:15:57,473 WARN [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionServer(3299): Not adding moved region record: 301a23263bb8ba2547c849aa87975635 to self. 2024-12-10T01:15:57,475 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,475 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=CLOSED 2024-12-10T01:15:57,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:57,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-10T01:15:57,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 in 213 msec 2024-12-10T01:15:57,479 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE; state=CLOSED, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=true 2024-12-10T01:15:57,630 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE because future has completed 2024-12-10T01:15:57,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:15:57,792 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,793 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:15:57,793 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,793 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:15:57,793 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,794 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,796 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,797 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:57,797 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName A 2024-12-10T01:15:57,799 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:57,800 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:57,801 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,802 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:57,802 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName B 2024-12-10T01:15:57,802 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:57,803 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:57,803 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,803 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:15:57,804 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 301a23263bb8ba2547c849aa87975635 columnFamilyName C 2024-12-10T01:15:57,804 DEBUG [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:57,804 INFO [StoreOpener-301a23263bb8ba2547c849aa87975635-1 {}] regionserver.HStore(327): Store=301a23263bb8ba2547c849aa87975635/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:15:57,804 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,804 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,805 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,806 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,807 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,807 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,807 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:15:57,808 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,809 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 301a23263bb8ba2547c849aa87975635; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66497319, jitterRate=-0.009112730622291565}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:15:57,809 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,810 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 301a23263bb8ba2547c849aa87975635: Running coprocessor pre-open hook at 1733793357794Writing region info on filesystem at 1733793357794Initializing all the Stores at 1733793357795 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793357795Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793357795Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793357795Cleaning up temporary data from old regions at 1733793357807 (+12 ms)Running coprocessor post-open hooks at 1733793357809 (+2 ms)Region opened successfully at 1733793357810 (+1 ms) 2024-12-10T01:15:57,811 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., pid=82, masterSystemTime=1733793357786 2024-12-10T01:15:57,812 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,812 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:57,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=OPEN, openSeqNum=5, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:15:57,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=80 2024-12-10T01:15:57,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 in 182 msec 2024-12-10T01:15:57,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-10T01:15:57,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, REOPEN/MOVE in 556 msec 2024-12-10T01:15:57,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-10T01:15:57,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 562 msec 2024-12-10T01:15:57,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 984 msec 2024-12-10T01:15:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T01:15:57,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:15:57,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a9751d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,823 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3f877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fb311b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346078b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,826 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dab833c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,827 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3094a8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c26e048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216afcc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22de9dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a9be1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,832 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ecf134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,832 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe51192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20042efd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e55f07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd84be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f70d2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,836 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc66da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,837 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31209117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739e85a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,839 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18edc343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,840 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1883ac66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de09a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d88e577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71529c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73108557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457be81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ff3fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44310619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:15:57,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:15:57,849 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e575652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:15:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:15:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:15:57,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:15:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309ba95d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:15:57,851 DEBUG [Thread-1406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,851 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,851 DEBUG [Thread-1419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,852 DEBUG [Thread-1402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,852 DEBUG [Thread-1415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,852 DEBUG [Thread-1417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,852 DEBUG [Thread-1413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:15:57,854 DEBUG [Thread-1408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,855 DEBUG [Thread-1404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,855 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:15:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,856 DEBUG [Thread-1410 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-12-10T01:15:57,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,856 DEBUG [Thread-1410 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,859 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:15:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,865 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:15:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:15:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:15:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:15:57,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:15:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:57,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bdac80b061a54310bc0de119aa9605ac_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793357873/Put/seqid=0 2024-12-10T01:15:57,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733793357901, max timestamp=1733793357913Num uniques 859; 2024-12-10T01:15:57,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733793357901, max timestamp=1733793357913Num uniques 859; 2024-12-10T01:15:57,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:57,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:57,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793357901, max timestamp=1733793357913Num uniques 859; 2024-12-10T01:15:57,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:57,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742129_1305 (size=9714) 2024-12-10T01:15:57,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:57,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bdac80b061a54310bc0de119aa9605ac_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bdac80b061a54310bc0de119aa9605ac_301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:57,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/472853891f924f9e98861ed94cf4ac04, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:15:57,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/472853891f924f9e98861ed94cf4ac04 is 175, key is test_row_0/A:col10/1733793357873/Put/seqid=0 2024-12-10T01:15:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742130_1306 (size=22361) 2024-12-10T01:15:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793417941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793417940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793417951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:57,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793417952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793417952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:15:58,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793418060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793418059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793418060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793418069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793418069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:15:58,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793418270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793418270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793418270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793418279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793418279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/472853891f924f9e98861ed94cf4ac04 2024-12-10T01:15:58,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/5bac26f3c49e4df0aa34434a6a81e23e is 50, key is test_row_0/B:col10/1733793357873/Put/seqid=0 2024-12-10T01:15:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742131_1307 (size=9657) 2024-12-10T01:15:58,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:15:58,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793418580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793418580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793418580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793418589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793418589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:58,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/5bac26f3c49e4df0aa34434a6a81e23e 2024-12-10T01:15:58,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:15:58,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9f39f199b177420abfb73ac2d3ff36e8 is 50, key is test_row_0/C:col10/1733793357873/Put/seqid=0 2024-12-10T01:15:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742132_1308 (size=9657) 2024-12-10T01:15:58,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9f39f199b177420abfb73ac2d3ff36e8 2024-12-10T01:15:58,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/472853891f924f9e98861ed94cf4ac04 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04 2024-12-10T01:15:58,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04, entries=100, sequenceid=15, filesize=21.8 K 2024-12-10T01:15:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/5bac26f3c49e4df0aa34434a6a81e23e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e 2024-12-10T01:15:58,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e, entries=100, sequenceid=15, filesize=9.4 K 2024-12-10T01:15:58,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9f39f199b177420abfb73ac2d3ff36e8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8 2024-12-10T01:15:58,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8, entries=100, sequenceid=15, filesize=9.4 K 2024-12-10T01:15:58,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 301a23263bb8ba2547c849aa87975635 in 942ms, sequenceid=15, compaction requested=false 2024-12-10T01:15:58,816 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T01:15:58,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:15:58,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T01:15:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:58,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:15:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f0d288d56bb406fb3d8a549a79a1d64_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793357919/Put/seqid=0 2024-12-10T01:15:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742133_1309 (size=12154) 2024-12-10T01:15:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:58,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f0d288d56bb406fb3d8a549a79a1d64_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f0d288d56bb406fb3d8a549a79a1d64_301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d9cd123f6f5d403281c9acc9d6b01e53, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:15:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d9cd123f6f5d403281c9acc9d6b01e53 is 175, key is test_row_0/A:col10/1733793357919/Put/seqid=0 2024-12-10T01:15:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742134_1310 (size=30955) 2024-12-10T01:15:58,969 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d9cd123f6f5d403281c9acc9d6b01e53 2024-12-10T01:15:58,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/32dadf92881f498c825909a4bd019ee7 is 50, key is test_row_0/B:col10/1733793357919/Put/seqid=0 2024-12-10T01:15:58,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742135_1311 (size=12001) 2024-12-10T01:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:15:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:59,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:15:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793419106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793419108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793419108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793419109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793419110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793419220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793419220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793419229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793419229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/32dadf92881f498c825909a4bd019ee7 2024-12-10T01:15:59,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/b03d7391e1534fb2b19d8f6058406163 is 50, key is test_row_0/C:col10/1733793357919/Put/seqid=0 2024-12-10T01:15:59,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742136_1312 (size=12001) 2024-12-10T01:15:59,396 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/b03d7391e1534fb2b19d8f6058406163 2024-12-10T01:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d9cd123f6f5d403281c9acc9d6b01e53 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53 2024-12-10T01:15:59,403 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53, entries=150, sequenceid=52, filesize=30.2 K 2024-12-10T01:15:59,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/32dadf92881f498c825909a4bd019ee7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7 2024-12-10T01:15:59,410 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T01:15:59,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/b03d7391e1534fb2b19d8f6058406163 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163 2024-12-10T01:15:59,415 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T01:15:59,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 301a23263bb8ba2547c849aa87975635 in 480ms, sequenceid=52, compaction requested=false 2024-12-10T01:15:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:15:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:15:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-10T01:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-12-10T01:15:59,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-10T01:15:59,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-12-10T01:15:59,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 1.5670 sec 2024-12-10T01:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:59,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:15:59,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:15:59,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ca25eff7dc594d6e9006982794f6c6d9_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793359104/Put/seqid=0 2024-12-10T01:15:59,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742137_1313 (size=14594) 2024-12-10T01:15:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793419459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793359455, max timestamp=1733793359467Num uniques 859; 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793359455, max timestamp=1733793359467Num uniques 859; 2024-12-10T01:15:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793359455, max timestamp=1733793359467Num uniques 859; 2024-12-10T01:15:59,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:15:59,512 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:15:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793419501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793419506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793419514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793419579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793419620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793419620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793419629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793419789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793419830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793419830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:15:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793419839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:15:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:15:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:15:59,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ca25eff7dc594d6e9006982794f6c6d9_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ca25eff7dc594d6e9006982794f6c6d9_301a23263bb8ba2547c849aa87975635 2024-12-10T01:15:59,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/c8411a49d4db46aabc924125236cc0cf, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:15:59,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/c8411a49d4db46aabc924125236cc0cf is 175, key is test_row_0/A:col10/1733793359104/Put/seqid=0 2024-12-10T01:15:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742138_1314 (size=39549) 2024-12-10T01:16:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T01:16:00,010 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:00,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-12-10T01:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:00,011 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:00,012 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:00,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:00,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793420099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:00,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793420129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,137 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793420140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793420140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793420149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/c8411a49d4db46aabc924125236cc0cf 2024-12-10T01:16:00,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/22f6eae235c2460fbfb67a7b74ae6fc0 is 50, key is test_row_0/B:col10/1733793359104/Put/seqid=0 2024-12-10T01:16:00,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742139_1315 (size=12001) 2024-12-10T01:16:00,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:00,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793420619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:00,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/22f6eae235c2460fbfb67a7b74ae6fc0 2024-12-10T01:16:00,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793420659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793420659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793420660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/665c114e08dd4183a9d94590fc7fdfe1 is 50, key is test_row_0/C:col10/1733793359104/Put/seqid=0 2024-12-10T01:16:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:00,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742140_1316 (size=12001) 2024-12-10T01:16:00,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:00,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:01,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:01,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:01,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:01,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/665c114e08dd4183a9d94590fc7fdfe1 2024-12-10T01:16:01,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/c8411a49d4db46aabc924125236cc0cf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf 2024-12-10T01:16:01,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf, entries=200, sequenceid=62, filesize=38.6 K 2024-12-10T01:16:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/22f6eae235c2460fbfb67a7b74ae6fc0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0 2024-12-10T01:16:01,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T01:16:01,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/665c114e08dd4183a9d94590fc7fdfe1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1 2024-12-10T01:16:01,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T01:16:01,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 301a23263bb8ba2547c849aa87975635 in 1667ms, sequenceid=62, compaction requested=true 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:01,099 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:01,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T01:16:01,099 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:01,100 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:01,100 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/B is initiating minor compaction (all files) 2024-12-10T01:16:01,100 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/B in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,100 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=32.9 K 2024-12-10T01:16:01,101 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bac26f3c49e4df0aa34434a6a81e23e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793357873 2024-12-10T01:16:01,101 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:01,101 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/A is initiating minor compaction (all files) 2024-12-10T01:16:01,101 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/A in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,101 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=90.7 K 2024-12-10T01:16:01,101 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,101 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf] 2024-12-10T01:16:01,102 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 472853891f924f9e98861ed94cf4ac04, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793357873 2024-12-10T01:16:01,102 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32dadf92881f498c825909a4bd019ee7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793357919 2024-12-10T01:16:01,102 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d9cd123f6f5d403281c9acc9d6b01e53, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793357919 2024-12-10T01:16:01,102 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f6eae235c2460fbfb67a7b74ae6fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:01,102 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c8411a49d4db46aabc924125236cc0cf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:01,109 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:01,112 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412101b9c76024e1440feba2017a0994e2ebe_301a23263bb8ba2547c849aa87975635 store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:01,113 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:01,113 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/994edde70f484f67bf0ff56e9161c41f is 50, key is test_row_0/B:col10/1733793359104/Put/seqid=0 2024-12-10T01:16:01,114 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412101b9c76024e1440feba2017a0994e2ebe_301a23263bb8ba2547c849aa87975635, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:01,114 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101b9c76024e1440feba2017a0994e2ebe_301a23263bb8ba2547c849aa87975635 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:01,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742141_1317 (size=12104) 2024-12-10T01:16:01,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742142_1318 (size=4469) 2024-12-10T01:16:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:01,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T01:16:01,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,231 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T01:16:01,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210defe99a2d6b04f74aa67e91993fd816f_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793359496/Put/seqid=0 2024-12-10T01:16:01,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742143_1319 (size=12154) 2024-12-10T01:16:01,523 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/994edde70f484f67bf0ff56e9161c41f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/994edde70f484f67bf0ff56e9161c41f 2024-12-10T01:16:01,527 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#A#compaction#270 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:01,528 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/521c17581d6f4f7ba571e151d0106eea is 175, key is test_row_0/A:col10/1733793359104/Put/seqid=0 2024-12-10T01:16:01,533 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/B of 301a23263bb8ba2547c849aa87975635 into 994edde70f484f67bf0ff56e9161c41f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:01,533 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:01,533 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/B, priority=13, startTime=1733793361099; duration=0sec 2024-12-10T01:16:01,533 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:01,533 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:B 2024-12-10T01:16:01,533 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:01,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742144_1320 (size=31058) 2024-12-10T01:16:01,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:01,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/C is initiating minor compaction (all files) 2024-12-10T01:16:01,535 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/C in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:01,535 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=32.9 K 2024-12-10T01:16:01,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f39f199b177420abfb73ac2d3ff36e8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733793357873 2024-12-10T01:16:01,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03d7391e1534fb2b19d8f6058406163, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733793357919 2024-12-10T01:16:01,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665c114e08dd4183a9d94590fc7fdfe1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:01,541 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#C#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:01,541 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/f0299dc421be4d43834bca83a7bf1dff is 50, key is test_row_0/C:col10/1733793359104/Put/seqid=0 2024-12-10T01:16:01,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742145_1321 (size=12104) 2024-12-10T01:16:01,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:01,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:01,649 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210defe99a2d6b04f74aa67e91993fd816f_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210defe99a2d6b04f74aa67e91993fd816f_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/fdc5f197f3ed437986c9a84717adcb7e, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/fdc5f197f3ed437986c9a84717adcb7e is 175, key is test_row_0/A:col10/1733793359496/Put/seqid=0 2024-12-10T01:16:01,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742146_1322 (size=30955) 2024-12-10T01:16:01,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793421682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793421685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793421689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793421690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,695 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,695 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793421800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793421800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:01,938 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/521c17581d6f4f7ba571e151d0106eea as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea 2024-12-10T01:16:01,941 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/A of 301a23263bb8ba2547c849aa87975635 into 521c17581d6f4f7ba571e151d0106eea(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:01,941 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:01,942 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/A, priority=13, startTime=1733793361099; duration=0sec 2024-12-10T01:16:01,942 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:01,942 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:A 2024-12-10T01:16:01,947 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/f0299dc421be4d43834bca83a7bf1dff as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f0299dc421be4d43834bca83a7bf1dff 2024-12-10T01:16:01,951 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/C of 301a23263bb8ba2547c849aa87975635 into f0299dc421be4d43834bca83a7bf1dff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:01,951 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:01,951 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/C, priority=13, startTime=1733793361099; duration=0sec 2024-12-10T01:16:01,951 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:01,951 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:C 2024-12-10T01:16:02,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793422009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793422010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/fdc5f197f3ed437986c9a84717adcb7e 2024-12-10T01:16:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/43714a02554d41deb06e5de667f05e99 is 50, key is test_row_0/B:col10/1733793359496/Put/seqid=0 2024-12-10T01:16:02,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742147_1323 (size=12001) 2024-12-10T01:16:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793422160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,162 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793422319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793422319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/43714a02554d41deb06e5de667f05e99 2024-12-10T01:16:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/da6b71906f4345bbba82875393d4f391 is 50, key is test_row_0/C:col10/1733793359496/Put/seqid=0 2024-12-10T01:16:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742148_1324 (size=12001) 2024-12-10T01:16:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793422829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793422829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:02,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/da6b71906f4345bbba82875393d4f391 2024-12-10T01:16:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/fdc5f197f3ed437986c9a84717adcb7e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e 2024-12-10T01:16:02,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e, entries=150, sequenceid=98, filesize=30.2 K 2024-12-10T01:16:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/43714a02554d41deb06e5de667f05e99 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99 2024-12-10T01:16:02,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:16:02,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/da6b71906f4345bbba82875393d4f391 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391 2024-12-10T01:16:02,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391, entries=150, sequenceid=98, filesize=11.7 K 2024-12-10T01:16:02,956 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 301a23263bb8ba2547c849aa87975635 in 1725ms, sequenceid=98, compaction requested=false 2024-12-10T01:16:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-10T01:16:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-10T01:16:02,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-10T01:16:02,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9450 sec 2024-12-10T01:16:02,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.9490 sec 2024-12-10T01:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:03,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:03,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e857575b29a3428490c660067d2f6a96_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:03,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742149_1325 (size=14594) 2024-12-10T01:16:03,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:03,728 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e857575b29a3428490c660067d2f6a96_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e857575b29a3428490c660067d2f6a96_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:03,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/4c0287dee127481bacf9cf88d1a76fac, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:03,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/4c0287dee127481bacf9cf88d1a76fac is 175, key is test_row_0/A:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:03,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742150_1326 (size=39549) 2024-12-10T01:16:03,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793423774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793363740, max timestamp=1733793363774Num uniques 859; 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793363740, max timestamp=1733793363774Num uniques 859; 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793363740, max timestamp=1733793363774Num uniques 859; 2024-12-10T01:16:03,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793423862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793423865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793423869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793423879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793423979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793423979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793423989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793424099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,134 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/4c0287dee127481bacf9cf88d1a76fac 2024-12-10T01:16:04,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1694a86ca2f849969af3654441e203b8 is 50, key is test_row_0/B:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:04,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742151_1327 (size=12001) 2024-12-10T01:16:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-10T01:16:04,179 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:04,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-10T01:16:04,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:04,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:04,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:04,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793424189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793424189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793424199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:04,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793424409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:04,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793424499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793424499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793424509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1694a86ca2f849969af3654441e203b8 2024-12-10T01:16:04,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/2e29b54d8bea4679a613a4b24970e3c0 is 50, key is test_row_0/C:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:04,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742152_1328 (size=12001) 2024-12-10T01:16:04,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793424930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:04,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:04,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/2e29b54d8bea4679a613a4b24970e3c0 2024-12-10T01:16:04,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/4c0287dee127481bacf9cf88d1a76fac as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac 2024-12-10T01:16:04,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac, entries=200, sequenceid=112, filesize=38.6 K 2024-12-10T01:16:04,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1694a86ca2f849969af3654441e203b8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8 2024-12-10T01:16:04,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8, entries=150, sequenceid=112, filesize=11.7 K 2024-12-10T01:16:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/2e29b54d8bea4679a613a4b24970e3c0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0 2024-12-10T01:16:04,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0, entries=150, sequenceid=112, filesize=11.7 K 2024-12-10T01:16:04,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 301a23263bb8ba2547c849aa87975635 in 1255ms, sequenceid=112, compaction requested=true 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:04,969 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:04,969 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/B is initiating minor compaction (all files) 2024-12-10T01:16:04,970 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/B in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,970 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/994edde70f484f67bf0ff56e9161c41f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=35.3 K 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/A is initiating minor compaction (all files) 2024-12-10T01:16:04,970 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/A in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 994edde70f484f67bf0ff56e9161c41f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:04,970 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=99.2 K 2024-12-10T01:16:04,970 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac] 2024-12-10T01:16:04,970 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 43714a02554d41deb06e5de667f05e99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793359496 2024-12-10T01:16:04,971 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521c17581d6f4f7ba571e151d0106eea, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:04,971 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1694a86ca2f849969af3654441e203b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:04,972 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc5f197f3ed437986c9a84717adcb7e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793359496 2024-12-10T01:16:04,973 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0287dee127481bacf9cf88d1a76fac, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:04,978 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:04,979 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:04,979 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412102e768c30f0a349e2814d38de3f59299e_301a23263bb8ba2547c849aa87975635 store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:04,979 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/0c8fffe3de564610907b740a051f18c0 is 50, key is test_row_0/B:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:04,980 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412102e768c30f0a349e2814d38de3f59299e_301a23263bb8ba2547c849aa87975635, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:04,980 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102e768c30f0a349e2814d38de3f59299e_301a23263bb8ba2547c849aa87975635 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742153_1329 (size=12207) 2024-12-10T01:16:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742154_1330 (size=4469) 2024-12-10T01:16:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:05,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:05,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106a170ff6c2b841a4927790ff50631bd8_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793363851/Put/seqid=0 2024-12-10T01:16:05,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742155_1331 (size=17284) 2024-12-10T01:16:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793425041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793425044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793425050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793425160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793425160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793425169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793425379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793425379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793425379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,387 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#A#compaction#280 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:05,387 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/bdd61fc75fa24e1580c219bc38551684 is 175, key is test_row_0/A:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:05,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742156_1332 (size=31161) 2024-12-10T01:16:05,391 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/0c8fffe3de564610907b740a051f18c0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0c8fffe3de564610907b740a051f18c0 2024-12-10T01:16:05,394 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/bdd61fc75fa24e1580c219bc38551684 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684 2024-12-10T01:16:05,395 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/B of 301a23263bb8ba2547c849aa87975635 into 0c8fffe3de564610907b740a051f18c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:05,395 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:05,395 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/B, priority=13, startTime=1733793364969; duration=0sec 2024-12-10T01:16:05,395 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:05,395 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:B 2024-12-10T01:16:05,395 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:05,396 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:05,396 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/C is initiating minor compaction (all files) 2024-12-10T01:16:05,396 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/C in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,396 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f0299dc421be4d43834bca83a7bf1dff, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=35.3 K 2024-12-10T01:16:05,397 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f0299dc421be4d43834bca83a7bf1dff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733793359104 2024-12-10T01:16:05,397 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting da6b71906f4345bbba82875393d4f391, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733793359496 2024-12-10T01:16:05,398 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e29b54d8bea4679a613a4b24970e3c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:05,398 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/A of 301a23263bb8ba2547c849aa87975635 into bdd61fc75fa24e1580c219bc38551684(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:05,398 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:05,398 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/A, priority=13, startTime=1733793364969; duration=0sec 2024-12-10T01:16:05,398 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:05,398 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:A 2024-12-10T01:16:05,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,403 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:05,404 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/4f5cf744802142fabec45a975d53316e is 50, key is test_row_0/C:col10/1733793361668/Put/seqid=0 2024-12-10T01:16:05,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742157_1333 (size=12207) 2024-12-10T01:16:05,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:05,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106a170ff6c2b841a4927790ff50631bd8_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a170ff6c2b841a4927790ff50631bd8_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:05,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/939948ff3bec44ef860f0b02a39864b8, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:05,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/939948ff3bec44ef860f0b02a39864b8 is 175, key is test_row_0/A:col10/1733793363851/Put/seqid=0 2024-12-10T01:16:05,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742158_1334 (size=48389) 2024-12-10T01:16:05,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/939948ff3bec44ef860f0b02a39864b8 2024-12-10T01:16:05,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f550f82e76f540169fccf5735a400208 is 50, key is test_row_0/B:col10/1733793363851/Put/seqid=0 2024-12-10T01:16:05,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742159_1335 (size=12151) 2024-12-10T01:16:05,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793425689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793425689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793425700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:05,816 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/4f5cf744802142fabec45a975d53316e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4f5cf744802142fabec45a975d53316e 2024-12-10T01:16:05,819 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 301a23263bb8ba2547c849aa87975635/C of 301a23263bb8ba2547c849aa87975635 into 4f5cf744802142fabec45a975d53316e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:05,819 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:05,819 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/C, priority=13, startTime=1733793364969; duration=0sec 2024-12-10T01:16:05,820 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:05,820 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:C 2024-12-10T01:16:05,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:05,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:05,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f550f82e76f540169fccf5735a400208 2024-12-10T01:16:05,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/be4586ca9a46415d99f77dd0cf045705 is 50, key is test_row_0/C:col10/1733793363851/Put/seqid=0 2024-12-10T01:16:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742160_1336 (size=12151) 2024-12-10T01:16:05,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793425950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:05,953 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:06,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:06,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:06,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793426179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:06,183 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:06,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793426209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793426210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:06,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793426219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:06,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/be4586ca9a46415d99f77dd0cf045705 2024-12-10T01:16:06,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/939948ff3bec44ef860f0b02a39864b8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8 2024-12-10T01:16:06,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8, entries=250, sequenceid=149, filesize=47.3 K 2024-12-10T01:16:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f550f82e76f540169fccf5735a400208 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208 2024-12-10T01:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208, entries=150, sequenceid=149, filesize=11.9 K 2024-12-10T01:16:06,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/be4586ca9a46415d99f77dd0cf045705 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705 2024-12-10T01:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705, entries=150, sequenceid=149, filesize=11.9 K 2024-12-10T01:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for 301a23263bb8ba2547c849aa87975635 in 1275ms, sequenceid=149, compaction requested=false 2024-12-10T01:16:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:06,318 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103a057e7f71f34a57b26867eb154fafd3_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_1/A:col10/1733793365049/Put/seqid=0 2024-12-10T01:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742161_1337 (size=9814) 2024-12-10T01:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,737 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103a057e7f71f34a57b26867eb154fafd3_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a057e7f71f34a57b26867eb154fafd3_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/79e5170c456e46d08fc324394dd99318, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/79e5170c456e46d08fc324394dd99318 is 175, key is test_row_1/A:col10/1733793365049/Put/seqid=0 2024-12-10T01:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742162_1338 (size=22461) 2024-12-10T01:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,151 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/79e5170c456e46d08fc324394dd99318 2024-12-10T01:16:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/ff56256682c444c7a7150115c934161b is 50, key is test_row_1/B:col10/1733793365049/Put/seqid=0 2024-12-10T01:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742163_1339 (size=9757) 2024-12-10T01:16:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793367321, max timestamp=1733793367335Num uniques 859; 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793367321, max timestamp=1733793367335Num uniques 859; 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793367321, max timestamp=1733793367335Num uniques 859; 2024-12-10T01:16:07,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793367340, max timestamp=1733793367413Num uniques 1068; 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793367340, max timestamp=1733793367413Num uniques 1068; 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793367340, max timestamp=1733793367413Num uniques 1068; 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T01:16:07,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T01:16:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793427460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793427467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793427468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793427579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793427579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793427579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/ff56256682c444c7a7150115c934161b 2024-12-10T01:16:07,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fabae456288d44d08a400a9b70ca68a2 is 50, key is test_row_1/C:col10/1733793365049/Put/seqid=0 2024-12-10T01:16:07,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742164_1340 (size=9757) 2024-12-10T01:16:07,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793427799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793427799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793427799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793427979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:07,985 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fabae456288d44d08a400a9b70ca68a2 2024-12-10T01:16:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/79e5170c456e46d08fc324394dd99318 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318 2024-12-10T01:16:08,046 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318, entries=100, sequenceid=161, filesize=21.9 K 2024-12-10T01:16:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/ff56256682c444c7a7150115c934161b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b 2024-12-10T01:16:08,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b, entries=100, sequenceid=161, filesize=9.5 K 2024-12-10T01:16:08,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fabae456288d44d08a400a9b70ca68a2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2 2024-12-10T01:16:08,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2, entries=100, sequenceid=161, filesize=9.5 K 2024-12-10T01:16:08,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 301a23263bb8ba2547c849aa87975635 in 1739ms, sequenceid=161, compaction requested=true 2024-12-10T01:16:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-10T01:16:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-10T01:16:08,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-10T01:16:08,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8770 sec 2024-12-10T01:16:08,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.8800 sec 2024-12-10T01:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:08,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=145.36 KB heapSize=414.92 KB 2024-12-10T01:16:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793367451, max timestamp=1733793368116Num uniques 1077; 2024-12-10T01:16:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T01:16:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793367451, max timestamp=1733793368116Num uniques 1077; 2024-12-10T01:16:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:08,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793367451, max timestamp=1733793368116Num uniques 1027; 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:08,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:08,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:08,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc4565bc3b3f4fd483b23a8ba9b15959_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:08,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742165_1341 (size=12304) 2024-12-10T01:16:08,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:08,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc4565bc3b3f4fd483b23a8ba9b15959_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc4565bc3b3f4fd483b23a8ba9b15959_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:08,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/df3ab0b67b2345b6bb2b90f7bbed8ed6, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:08,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 is 175, key is test_row_0/A:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:08,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742166_1342 (size=31105) 2024-12-10T01:16:08,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 2024-12-10T01:16:08,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/c1f31bd37e164f2a9a9b5f64abdb30f6 is 50, key is test_row_0/B:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742167_1343 (size=12151) 2024-12-10T01:16:08,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793368155, max timestamp=1733793368203Num uniques 859; 2024-12-10T01:16:08,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793368155, max timestamp=1733793368203Num uniques 859; 2024-12-10T01:16:08,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:08,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:08,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793368155, max timestamp=1733793368203Num uniques 859; 2024-12-10T01:16:08,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793368238, max timestamp=1733793368265Num uniques 1018; 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733793368238, max timestamp=1733793368265Num uniques 1018; 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793368249, max timestamp=1733793368265Num uniques 1018; 2024-12-10T01:16:08,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T01:16:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733793368300, max timestamp=1733793368325Num uniques 1027; 2024-12-10T01:16:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733793368300, max timestamp=1733793368325Num uniques 1077; 2024-12-10T01:16:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-10T01:16:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T01:16:08,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733793368300, max timestamp=1733793368325Num uniques 1077; 2024-12-10T01:16:08,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-10T01:16:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-10T01:16:08,350 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:08,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-10T01:16:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:08,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:08,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:08,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733793368356, max timestamp=1733793368385Num uniques 1086; 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733793368356, max timestamp=1733793368385Num uniques 1086; 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733793368356, max timestamp=1733793368385Num uniques 1086; 2024-12-10T01:16:08,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T01:16:08,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733793368408, max timestamp=1733793368443Num uniques 1145; 2024-12-10T01:16:08,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-10T01:16:08,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733793368408, max timestamp=1733793368443Num uniques 1145; 2024-12-10T01:16:08,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-10T01:16:08,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733793368408, max timestamp=1733793368443Num uniques 1145; 2024-12-10T01:16:08,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-10T01:16:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:08,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T01:16:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793428508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793428519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793428521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/c1f31bd37e164f2a9a9b5f64abdb30f6 2024-12-10T01:16:08,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/3f35acc49ebd4fabaff96b28ce3e91e3 is 50, key is test_row_0/C:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:08,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742168_1344 (size=12151) 2024-12-10T01:16:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793428629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793428640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793428640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T01:16:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:08,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T01:16:08,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:08,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793428839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793428850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793428850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:08,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T01:16:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:08,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/3f35acc49ebd4fabaff96b28ce3e91e3 2024-12-10T01:16:08,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 2024-12-10T01:16:08,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6, entries=150, sequenceid=217, filesize=30.4 K 2024-12-10T01:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:08,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/c1f31bd37e164f2a9a9b5f64abdb30f6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6 2024-12-10T01:16:08,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6, entries=150, sequenceid=217, filesize=11.9 K 2024-12-10T01:16:08,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/3f35acc49ebd4fabaff96b28ce3e91e3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3 2024-12-10T01:16:08,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3, entries=150, sequenceid=217, filesize=11.9 K 2024-12-10T01:16:08,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~84.42 KB/86448, currentSize=154.31 KB/158010 for 301a23263bb8ba2547c849aa87975635 in 871ms, sequenceid=217, compaction requested=true 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:08,988 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:08,988 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:08,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:08,989 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:08,989 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:08,989 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/A is initiating minor compaction (all files) 2024-12-10T01:16:08,989 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/B is initiating minor compaction (all files) 2024-12-10T01:16:08,989 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/A in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,989 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/B in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,990 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0c8fffe3de564610907b740a051f18c0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=45.2 K 2024-12-10T01:16:08,990 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=130.0 K 2024-12-10T01:16:08,990 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6] 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8fffe3de564610907b740a051f18c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdd61fc75fa24e1580c219bc38551684, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f550f82e76f540169fccf5735a400208, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733793363851 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939948ff3bec44ef860f0b02a39864b8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733793363851 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ff56256682c444c7a7150115c934161b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733793365039 2024-12-10T01:16:08,990 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e5170c456e46d08fc324394dd99318, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733793365039 2024-12-10T01:16:08,991 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f31bd37e164f2a9a9b5f64abdb30f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:08,991 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting df3ab0b67b2345b6bb2b90f7bbed8ed6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:08,996 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:08,997 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:08,998 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/58ca659c6e5643fcb3c49007dc7da255 is 50, key is test_row_0/B:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:08,998 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210049824523de847bfa5ec9461ca2a9bd6_301a23263bb8ba2547c849aa87975635 store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:09,000 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210049824523de847bfa5ec9461ca2a9bd6_301a23263bb8ba2547c849aa87975635, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:09,001 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210049824523de847bfa5ec9461ca2a9bd6_301a23263bb8ba2547c849aa87975635 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:09,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742169_1345 (size=12493) 2024-12-10T01:16:09,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742170_1346 (size=4469) 2024-12-10T01:16:09,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T01:16:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:09,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210940e66f265ba4b50a45389a90b1dee21_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793368463/Put/seqid=0 2024-12-10T01:16:09,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742171_1347 (size=12454) 2024-12-10T01:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:09,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793429166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793429166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793429170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793429279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793429280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793429280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,406 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#A#compaction#292 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:09,407 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/406e307107e746a8963d1f6db2ad689c is 175, key is test_row_0/A:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:09,407 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/58ca659c6e5643fcb3c49007dc7da255 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/58ca659c6e5643fcb3c49007dc7da255 2024-12-10T01:16:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742172_1348 (size=31447) 2024-12-10T01:16:09,411 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/B of 301a23263bb8ba2547c849aa87975635 into 58ca659c6e5643fcb3c49007dc7da255(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:09,411 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:09,411 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/B, priority=12, startTime=1733793368988; duration=0sec 2024-12-10T01:16:09,411 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:09,411 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:B 2024-12-10T01:16:09,412 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:09,413 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:09,413 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/C is initiating minor compaction (all files) 2024-12-10T01:16:09,413 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/C in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:09,413 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4f5cf744802142fabec45a975d53316e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=45.2 K 2024-12-10T01:16:09,413 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f5cf744802142fabec45a975d53316e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733793361668 2024-12-10T01:16:09,413 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/406e307107e746a8963d1f6db2ad689c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c 2024-12-10T01:16:09,413 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting be4586ca9a46415d99f77dd0cf045705, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733793363851 2024-12-10T01:16:09,414 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting fabae456288d44d08a400a9b70ca68a2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733793365039 2024-12-10T01:16:09,414 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f35acc49ebd4fabaff96b28ce3e91e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:09,417 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/A of 301a23263bb8ba2547c849aa87975635 into 406e307107e746a8963d1f6db2ad689c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:09,417 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:09,417 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/A, priority=12, startTime=1733793368988; duration=0sec 2024-12-10T01:16:09,417 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:09,417 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:A 2024-12-10T01:16:09,422 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#C#compaction#294 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:09,423 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f is 50, key is test_row_0/C:col10/1733793367465/Put/seqid=0 2024-12-10T01:16:09,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742173_1349 (size=12493) 2024-12-10T01:16:09,433 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f 2024-12-10T01:16:09,436 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/C of 301a23263bb8ba2547c849aa87975635 into f5c8b3a3e4cf444a8b7ba6b72965ee8f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:09,436 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:09,436 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/C, priority=12, startTime=1733793368988; duration=0sec 2024-12-10T01:16:09,437 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:09,437 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:C 2024-12-10T01:16:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793429489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793429500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793429500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:09,527 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210940e66f265ba4b50a45389a90b1dee21_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210940e66f265ba4b50a45389a90b1dee21_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/23862f753084443e9121c8e05bd271a1, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/23862f753084443e9121c8e05bd271a1 is 175, key is test_row_0/A:col10/1733793368463/Put/seqid=0 2024-12-10T01:16:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742174_1350 (size=31255) 2024-12-10T01:16:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793429799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793429809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793429809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:09,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/23862f753084443e9121c8e05bd271a1 2024-12-10T01:16:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/111a47ce3ca04e10bdd1c42fee9f3878 is 50, key is test_row_0/B:col10/1733793368463/Put/seqid=0 2024-12-10T01:16:09,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742175_1351 (size=12301) 2024-12-10T01:16:09,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/111a47ce3ca04e10bdd1c42fee9f3878 2024-12-10T01:16:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9be567e4b2354ec88709f39c15c18cf9 is 50, key is test_row_0/C:col10/1733793368463/Put/seqid=0 2024-12-10T01:16:09,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742176_1352 (size=12301) 2024-12-10T01:16:10,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793430309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793430319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793430319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:10,362 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9be567e4b2354ec88709f39c15c18cf9 2024-12-10T01:16:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/23862f753084443e9121c8e05bd271a1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1 2024-12-10T01:16:10,370 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1, entries=150, sequenceid=323, filesize=30.5 K 2024-12-10T01:16:10,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/111a47ce3ca04e10bdd1c42fee9f3878 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878 2024-12-10T01:16:10,374 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878, entries=150, sequenceid=323, filesize=12.0 K 2024-12-10T01:16:10,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9be567e4b2354ec88709f39c15c18cf9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9 2024-12-10T01:16:10,378 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9, entries=150, sequenceid=323, filesize=12.0 K 2024-12-10T01:16:10,379 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for 301a23263bb8ba2547c849aa87975635 in 1265ms, sequenceid=323, compaction requested=false 2024-12-10T01:16:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-10T01:16:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-10T01:16:10,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-10T01:16:10,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0280 sec 2024-12-10T01:16:10,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.0340 sec 2024-12-10T01:16:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T01:16:10,499 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:10,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-10T01:16:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:10,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:10,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:10,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:10,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T01:16:10,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:10,653 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T01:16:10,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100ef95a80e6064809a84195f4d5de11fa_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793369165/Put/seqid=0 2024-12-10T01:16:10,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742177_1353 (size=12454) 2024-12-10T01:16:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:11,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:11,069 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100ef95a80e6064809a84195f4d5de11fa_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100ef95a80e6064809a84195f4d5de11fa_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:11,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/a54b1d8c367e4b06b9282c01e9e0fd5a, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:11,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/a54b1d8c367e4b06b9282c01e9e0fd5a is 175, key is test_row_0/A:col10/1733793369165/Put/seqid=0 2024-12-10T01:16:11,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742178_1354 (size=31255) 2024-12-10T01:16:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:11,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793371340, max timestamp=1733793371372Num uniques 859; 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793371340, max timestamp=1733793371372Num uniques 859; 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733793371340, max timestamp=1733793371372Num uniques 859; 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:16:11,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T01:16:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793431420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793431421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793371404, max timestamp=1733793371428Num uniques 1168; 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793371404, max timestamp=1733793371428Num uniques 1168; 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733793371404, max timestamp=1733793371428Num uniques 1168; 2024-12-10T01:16:11,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-10T01:16:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:11,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/a54b1d8c367e4b06b9282c01e9e0fd5a 2024-12-10T01:16:11,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1b1ad5b8f1ef4a188b144f393ad1c632 is 50, key is test_row_0/B:col10/1733793369165/Put/seqid=0 2024-12-10T01:16:11,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742179_1355 (size=12301) 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793371458, max timestamp=1733793371541Num uniques 1027; 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793371458, max timestamp=1733793371541Num uniques 1027; 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733793371458, max timestamp=1733793371541Num uniques 1027; 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T01:16:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T01:16:11,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793371572, max timestamp=1733793371598Num uniques 1036; 2024-12-10T01:16:11,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793371572, max timestamp=1733793371598Num uniques 1036; 2024-12-10T01:16:11,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T01:16:11,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T01:16:11,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733793371572, max timestamp=1733793371598Num uniques 1036; 2024-12-10T01:16:11,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T01:16:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:11,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793371634, max timestamp=1733793371643Num uniques 1045; 2024-12-10T01:16:11,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:16:11,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793371634, max timestamp=1733793371643Num uniques 1045; 2024-12-10T01:16:11,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:16:11,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733793371634, max timestamp=1733793371643Num uniques 1045; 2024-12-10T01:16:11,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T01:16:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733793371682, max timestamp=1733793371698Num uniques 1104; 2024-12-10T01:16:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-10T01:16:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733793371682, max timestamp=1733793371698Num uniques 1104; 2024-12-10T01:16:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-10T01:16:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733793371682, max timestamp=1733793371698Num uniques 1104; 2024-12-10T01:16:11,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-10T01:16:11,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793431757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:11,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793431757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:11,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733793371739, max timestamp=1733793371766Num uniques 1113; 2024-12-10T01:16:11,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733793371739, max timestamp=1733793371766Num uniques 1113; 2024-12-10T01:16:11,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-10T01:16:11,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-10T01:16:11,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733793371739, max timestamp=1733793371766Num uniques 1113; 2024-12-10T01:16:11,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-10T01:16:11,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1b1ad5b8f1ef4a188b144f393ad1c632 2024-12-10T01:16:11,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ee81a4ef883343b29e9f4166fbb0cfc1 is 50, key is test_row_0/C:col10/1733793369165/Put/seqid=0 2024-12-10T01:16:11,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742180_1356 (size=12301) 2024-12-10T01:16:11,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ee81a4ef883343b29e9f4166fbb0cfc1 2024-12-10T01:16:11,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/a54b1d8c367e4b06b9282c01e9e0fd5a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a 2024-12-10T01:16:11,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733793371849, max timestamp=1733793371903Num uniques 1072; 2024-12-10T01:16:11,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733793371849, max timestamp=1733793371903Num uniques 1072; 2024-12-10T01:16:11,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-10T01:16:11,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-10T01:16:11,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733793371849, max timestamp=1733793371903Num uniques 1072; 2024-12-10T01:16:11,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-10T01:16:11,909 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a, entries=150, sequenceid=333, filesize=30.5 K 2024-12-10T01:16:11,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/1b1ad5b8f1ef4a188b144f393ad1c632 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632 2024-12-10T01:16:11,913 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632, entries=150, sequenceid=333, filesize=12.0 K 2024-12-10T01:16:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ee81a4ef883343b29e9f4166fbb0cfc1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1 2024-12-10T01:16:11,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1, entries=150, sequenceid=333, filesize=12.0 K 2024-12-10T01:16:11,918 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=40.25 KB/41220 for 301a23263bb8ba2547c849aa87975635 in 1265ms, sequenceid=333, compaction requested=true 2024-12-10T01:16:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-10T01:16:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-10T01:16:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:11,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=53.67 KB heapSize=144.70 KB 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:11,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-10T01:16:11,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-12-10T01:16:11,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.4270 sec 2024-12-10T01:16:11,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b140bef0eed74740b7b6677d4d084459_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742181_1357 (size=17534) 2024-12-10T01:16:11,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793431973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793371954, max timestamp=1733793371974Num uniques 859; 2024-12-10T01:16:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793371954, max timestamp=1733793371974Num uniques 859; 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793371954, max timestamp=1733793371974Num uniques 859; 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:11,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:12,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793432023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793432033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793432035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793432079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793432140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793432149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793432159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793432289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:12,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b140bef0eed74740b7b6677d4d084459_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b140bef0eed74740b7b6677d4d084459_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:12,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d3ee98e23f6646f2b8094732cc581cca, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:12,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d3ee98e23f6646f2b8094732cc581cca is 175, key is test_row_0/A:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:12,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742182_1358 (size=48639) 2024-12-10T01:16:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793432360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793432369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793432379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793432600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T01:16:12,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:12,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:12,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793432680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793432689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793432699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-10T01:16:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:12,709 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:12,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:12,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:12,741 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d3ee98e23f6646f2b8094732cc581cca 2024-12-10T01:16:12,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/3adab1591821468da130174b78ff1af9 is 50, key is test_row_0/B:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:12,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742183_1359 (size=12301) 2024-12-10T01:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:12,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:12,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793433119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/3adab1591821468da130174b78ff1af9 2024-12-10T01:16:13,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ba72a5a560234862ba72ea2e6ba8087b is 50, key is test_row_0/C:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:13,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742184_1360 (size=12301) 2024-12-10T01:16:13,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ba72a5a560234862ba72ea2e6ba8087b 2024-12-10T01:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/d3ee98e23f6646f2b8094732cc581cca as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca 2024-12-10T01:16:13,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca, entries=250, sequenceid=479, filesize=47.5 K 2024-12-10T01:16:13,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/3adab1591821468da130174b78ff1af9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9 2024-12-10T01:16:13,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9, entries=150, sequenceid=479, filesize=12.0 K 2024-12-10T01:16:13,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/ba72a5a560234862ba72ea2e6ba8087b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b 2024-12-10T01:16:13,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b, entries=150, sequenceid=479, filesize=12.0 K 2024-12-10T01:16:13,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.58 KB/168528, currentSize=127.47 KB/130530 for 301a23263bb8ba2547c849aa87975635 in 1263ms, sequenceid=479, compaction requested=true 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:13,182 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:13,182 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:13,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:13,183 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:13,183 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/B is initiating minor compaction (all files) 2024-12-10T01:16:13,184 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/B in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,184 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/58ca659c6e5643fcb3c49007dc7da255, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=48.2 K 2024-12-10T01:16:13,184 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142596 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:13,184 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/A is initiating minor compaction (all files) 2024-12-10T01:16:13,184 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/A in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,184 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=139.3 K 2024-12-10T01:16:13,184 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,184 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca] 2024-12-10T01:16:13,184 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ca659c6e5643fcb3c49007dc7da255, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:13,185 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406e307107e746a8963d1f6db2ad689c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:13,185 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 111a47ce3ca04e10bdd1c42fee9f3878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733793368463 2024-12-10T01:16:13,185 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23862f753084443e9121c8e05bd271a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733793368463 2024-12-10T01:16:13,185 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1ad5b8f1ef4a188b144f393ad1c632, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733793369165 2024-12-10T01:16:13,185 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54b1d8c367e4b06b9282c01e9e0fd5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733793369165 2024-12-10T01:16:13,186 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adab1591821468da130174b78ff1af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371918 2024-12-10T01:16:13,186 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ee98e23f6646f2b8094732cc581cca, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371849 2024-12-10T01:16:13,193 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:13,196 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:13,197 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/70c15e028dd74903bb1a1c981a9e98fc is 50, key is test_row_0/B:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:13,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-10T01:16:13,199 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412106774fcd24b1248a0bae59eb07a9c7827_301a23263bb8ba2547c849aa87975635 store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:13,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:13,202 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412106774fcd24b1248a0bae59eb07a9c7827_301a23263bb8ba2547c849aa87975635, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:13,202 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106774fcd24b1248a0bae59eb07a9c7827_301a23263bb8ba2547c849aa87975635 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:13,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742185_1361 (size=12779) 2024-12-10T01:16:13,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742186_1362 (size=4469) 2024-12-10T01:16:13,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121067c98513814a4d3a811b5035819e6003_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793372001/Put/seqid=0 2024-12-10T01:16:13,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742187_1363 (size=14994) 2024-12-10T01:16:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793433233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793433243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793433244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:13,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793433349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793433359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793433369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793433559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793433569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793433579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,622 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/70c15e028dd74903bb1a1c981a9e98fc as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/70c15e028dd74903bb1a1c981a9e98fc 2024-12-10T01:16:13,625 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/B of 301a23263bb8ba2547c849aa87975635 into 70c15e028dd74903bb1a1c981a9e98fc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:13,625 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:13,625 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/B, priority=12, startTime=1733793373182; duration=0sec 2024-12-10T01:16:13,626 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:13,626 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:B 2024-12-10T01:16:13,626 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:13,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:13,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/C is initiating minor compaction (all files) 2024-12-10T01:16:13,627 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/C in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,627 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=48.2 K 2024-12-10T01:16:13,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c8b3a3e4cf444a8b7ba6b72965ee8f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733793367465 2024-12-10T01:16:13,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be567e4b2354ec88709f39c15c18cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733793368463 2024-12-10T01:16:13,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ee81a4ef883343b29e9f4166fbb0cfc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733793369165 2024-12-10T01:16:13,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ba72a5a560234862ba72ea2e6ba8087b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371918 2024-12-10T01:16:13,635 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:13,635 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/4b8980a7b6a3486ab92507f732cd418b is 50, key is test_row_0/C:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:13,636 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#A#compaction#303 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:13,637 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b017dffdc9bb47cd90982230feb5ab71 is 175, key is test_row_0/A:col10/1733793371918/Put/seqid=0 2024-12-10T01:16:13,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:13,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742188_1364 (size=12779) 2024-12-10T01:16:13,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742189_1365 (size=31733) 2024-12-10T01:16:13,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121067c98513814a4d3a811b5035819e6003_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121067c98513814a4d3a811b5035819e6003_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:13,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/6484994d853b451589ec39025e9dee4d, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:13,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/6484994d853b451589ec39025e9dee4d is 175, key is test_row_0/A:col10/1733793372001/Put/seqid=0 2024-12-10T01:16:13,644 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/4b8980a7b6a3486ab92507f732cd418b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4b8980a7b6a3486ab92507f732cd418b 2024-12-10T01:16:13,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742190_1366 (size=39949) 2024-12-10T01:16:13,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=514, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/6484994d853b451589ec39025e9dee4d 2024-12-10T01:16:13,649 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/C of 301a23263bb8ba2547c849aa87975635 into 4b8980a7b6a3486ab92507f732cd418b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:13,649 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:13,649 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/C, priority=12, startTime=1733793373182; duration=0sec 2024-12-10T01:16:13,649 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:13,649 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:C 2024-12-10T01:16:13,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f6e542a98f254266bb2d0cd0ed84429f is 50, key is test_row_0/B:col10/1733793372001/Put/seqid=0 2024-12-10T01:16:13,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742191_1367 (size=12301) 2024-12-10T01:16:13,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793433870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793433879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793433889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:13,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:13,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,044 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b017dffdc9bb47cd90982230feb5ab71 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71 2024-12-10T01:16:14,048 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/A of 301a23263bb8ba2547c849aa87975635 into b017dffdc9bb47cd90982230feb5ab71(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:14,048 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:14,048 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/A, priority=12, startTime=1733793373182; duration=0sec 2024-12-10T01:16:14,048 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:14,048 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:A 2024-12-10T01:16:14,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f6e542a98f254266bb2d0cd0ed84429f 2024-12-10T01:16:14,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/94081604126f4ddeb219a899ab36bce2 is 50, key is test_row_0/C:col10/1733793372001/Put/seqid=0 2024-12-10T01:16:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742192_1368 (size=12301) 2024-12-10T01:16:14,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793434129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:14,135 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:14,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793434379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:14,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:14,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:14,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793434399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793434399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:14,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/94081604126f4ddeb219a899ab36bce2 2024-12-10T01:16:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/6484994d853b451589ec39025e9dee4d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d 2024-12-10T01:16:14,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d, entries=200, sequenceid=514, filesize=39.0 K 2024-12-10T01:16:14,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/f6e542a98f254266bb2d0cd0ed84429f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f 2024-12-10T01:16:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f, entries=150, sequenceid=514, filesize=12.0 K 2024-12-10T01:16:14,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/94081604126f4ddeb219a899ab36bce2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2 2024-12-10T01:16:14,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2, entries=150, sequenceid=514, filesize=12.0 K 2024-12-10T01:16:14,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 301a23263bb8ba2547c849aa87975635 in 1289ms, sequenceid=514, compaction requested=false 2024-12-10T01:16:14,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:14,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T01:16:14,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:14,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121086abe3bf5cfb4e7b90fc223f8f090a44_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793373231/Put/seqid=0 2024-12-10T01:16:14,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742193_1369 (size=12454) 2024-12-10T01:16:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:14,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121086abe3bf5cfb4e7b90fc223f8f090a44_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121086abe3bf5cfb4e7b90fc223f8f090a44_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/86bb1a3f04a4433d9acfa46ef1cd8cbf, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/86bb1a3f04a4433d9acfa46ef1cd8cbf is 175, key is test_row_0/A:col10/1733793373231/Put/seqid=0 2024-12-10T01:16:14,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742194_1370 (size=31255) 2024-12-10T01:16:15,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=528, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/86bb1a3f04a4433d9acfa46ef1cd8cbf 2024-12-10T01:16:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/d6283354999142d19fdfc7f4b0891a06 is 50, key is test_row_0/B:col10/1733793373231/Put/seqid=0 2024-12-10T01:16:15,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742195_1371 (size=12301) 2024-12-10T01:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:15,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:15,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793375430, max timestamp=1733793375441Num uniques 859; 2024-12-10T01:16:15,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:15,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793375430, max timestamp=1733793375441Num uniques 859; 2024-12-10T01:16:15,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:15,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793375430, max timestamp=1733793375441Num uniques 859; 2024-12-10T01:16:15,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:15,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793435475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793435478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793435479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793435589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793435600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793435600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/d6283354999142d19fdfc7f4b0891a06 2024-12-10T01:16:15,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/37aad8fe3ded4faface71acb6312f916 is 50, key is test_row_0/C:col10/1733793373231/Put/seqid=0 2024-12-10T01:16:15,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742196_1372 (size=12301) 2024-12-10T01:16:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793435799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793435809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793435809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793436109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793436119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793436129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733793436150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,155 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/37aad8fe3ded4faface71acb6312f916 2024-12-10T01:16:16,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/86bb1a3f04a4433d9acfa46ef1cd8cbf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf 2024-12-10T01:16:16,189 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf, entries=150, sequenceid=528, filesize=30.5 K 2024-12-10T01:16:16,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/d6283354999142d19fdfc7f4b0891a06 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06 2024-12-10T01:16:16,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06, entries=150, sequenceid=528, filesize=12.0 K 2024-12-10T01:16:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/37aad8fe3ded4faface71acb6312f916 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916 2024-12-10T01:16:16,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916, entries=150, sequenceid=528, filesize=12.0 K 2024-12-10T01:16:16,198 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 301a23263bb8ba2547c849aa87975635 in 1655ms, sequenceid=528, compaction requested=true 2024-12-10T01:16:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-10T01:16:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-10T01:16:16,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T01:16:16,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4890 sec 2024-12-10T01:16:16,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.5410 sec 2024-12-10T01:16:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:16,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:16,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108b57b4e98e42479282543110d2b320c1_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742197_1373 (size=14994) 2024-12-10T01:16:16,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793436332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793436439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793436619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793436629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793436640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793436649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:16,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108b57b4e98e42479282543110d2b320c1_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b57b4e98e42479282543110d2b320c1_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:16,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/74afc5e21ab34019ab189cfbe754cf62, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:16,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/74afc5e21ab34019ab189cfbe754cf62 is 175, key is test_row_0/A:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:16,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742198_1374 (size=39949) 2024-12-10T01:16:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T01:16:16,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:16,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-10T01:16:16,891 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:16,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:16,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793436959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:17,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=563, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/74afc5e21ab34019ab189cfbe754cf62 2024-12-10T01:16:17,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/91bdd91a5f2847c190241f6d7afb2436 is 50, key is test_row_0/B:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:17,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742199_1375 (size=12301) 2024-12-10T01:16:17,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:17,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36998 deadline: 1733793437469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:17,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/91bdd91a5f2847c190241f6d7afb2436 2024-12-10T01:16:17,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9c8b4edc47064f0bbaca0be079169469 is 50, key is test_row_0/C:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:17,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742200_1376 (size=12301) 2024-12-10T01:16:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37008 deadline: 1733793437640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36964 deadline: 1733793437640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:17,644 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:17,644 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:17,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36956 deadline: 1733793437659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:17,666 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in 301a23263bb8ba2547c849aa87975635 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:17,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:17,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:17,856 INFO [Thread-1417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1417. 2024-12-10T01:16:17,856 DEBUG [Thread-1417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:17,856 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,856 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:17,863 INFO [Thread-1415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1415. 2024-12-10T01:16:17,863 DEBUG [Thread-1415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:17,863 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,863 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:17,863 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-12-10T01:16:17,863 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:17,863 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,863 INFO [Thread-1419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1419. 2024-12-10T01:16:17,863 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,863 DEBUG [Thread-1419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:17,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:17,863 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,864 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:17,871 INFO [Thread-1413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1413. 2024-12-10T01:16:17,871 DEBUG [Thread-1413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:17,871 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,871 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:17,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:17,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9c8b4edc47064f0bbaca0be079169469 2024-12-10T01:16:17,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/74afc5e21ab34019ab189cfbe754cf62 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62 2024-12-10T01:16:17,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62, entries=200, sequenceid=563, filesize=39.0 K 2024-12-10T01:16:17,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/91bdd91a5f2847c190241f6d7afb2436 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436 2024-12-10T01:16:17,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436, entries=150, sequenceid=563, filesize=12.0 K 2024-12-10T01:16:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/9c8b4edc47064f0bbaca0be079169469 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469 2024-12-10T01:16:17,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469, entries=150, sequenceid=563, filesize=12.0 K 2024-12-10T01:16:17,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 301a23263bb8ba2547c849aa87975635 in 1664ms, sequenceid=563, compaction requested=true 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:17,940 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 301a23263bb8ba2547c849aa87975635:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:17,940 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:17,941 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:17,941 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:17,941 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/B is initiating minor compaction (all files) 2024-12-10T01:16:17,941 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/A is initiating minor compaction (all files) 2024-12-10T01:16:17,941 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/B in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,941 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/A in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,941 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/70c15e028dd74903bb1a1c981a9e98fc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=48.5 K 2024-12-10T01:16:17,942 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=139.5 K 2024-12-10T01:16:17,942 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62] 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c15e028dd74903bb1a1c981a9e98fc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371918 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b017dffdc9bb47cd90982230feb5ab71, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371918 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e542a98f254266bb2d0cd0ed84429f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733793372001 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6484994d853b451589ec39025e9dee4d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733793372001 2024-12-10T01:16:17,942 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d6283354999142d19fdfc7f4b0891a06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733793373207 2024-12-10T01:16:17,943 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86bb1a3f04a4433d9acfa46ef1cd8cbf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733793373207 2024-12-10T01:16:17,943 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 91bdd91a5f2847c190241f6d7afb2436, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733793375461 2024-12-10T01:16:17,943 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74afc5e21ab34019ab189cfbe754cf62, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733793375461 2024-12-10T01:16:17,949 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:17,951 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:17,951 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210dff844f6cf5347368c7004434461f041_301a23263bb8ba2547c849aa87975635 store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:17,951 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/a2ea893c26b24e9781e82f923bd3f3ff is 50, key is test_row_0/B:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:17,954 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210dff844f6cf5347368c7004434461f041_301a23263bb8ba2547c849aa87975635, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:17,954 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dff844f6cf5347368c7004434461f041_301a23263bb8ba2547c849aa87975635 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:17,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742201_1377 (size=12915) 2024-12-10T01:16:17,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742202_1378 (size=4469) 2024-12-10T01:16:17,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:17,960 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d541c9a20fa84e3c8ba03c7e4a5a0aab_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793376320/Put/seqid=0 2024-12-10T01:16:17,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742203_1379 (size=12454) 2024-12-10T01:16:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:18,359 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:18,360 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/58d8bd2ac56542a8a354ffbfc5b52590 is 175, key is test_row_0/A:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:18,363 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/a2ea893c26b24e9781e82f923bd3f3ff as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/a2ea893c26b24e9781e82f923bd3f3ff 2024-12-10T01:16:18,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742204_1380 (size=31869) 2024-12-10T01:16:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:18,369 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/B of 301a23263bb8ba2547c849aa87975635 into a2ea893c26b24e9781e82f923bd3f3ff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:18,369 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:18,370 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/B, priority=12, startTime=1733793377940; duration=0sec 2024-12-10T01:16:18,370 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:18,370 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:B 2024-12-10T01:16:18,370 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:18,371 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:18,371 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): 301a23263bb8ba2547c849aa87975635/C is initiating minor compaction (all files) 2024-12-10T01:16:18,371 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 301a23263bb8ba2547c849aa87975635/C in TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:18,372 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4b8980a7b6a3486ab92507f732cd418b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp, totalSize=48.5 K 2024-12-10T01:16:18,373 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8980a7b6a3486ab92507f732cd418b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733793371918 2024-12-10T01:16:18,373 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 94081604126f4ddeb219a899ab36bce2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1733793372001 2024-12-10T01:16:18,373 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 37aad8fe3ded4faface71acb6312f916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733793373207 2024-12-10T01:16:18,373 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8b4edc47064f0bbaca0be079169469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733793375461 2024-12-10T01:16:18,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d541c9a20fa84e3c8ba03c7e4a5a0aab_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d541c9a20fa84e3c8ba03c7e4a5a0aab_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:18,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/96195273ee54499da44e5067fbad2001, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:18,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/96195273ee54499da44e5067fbad2001 is 175, key is test_row_0/A:col10/1733793376320/Put/seqid=0 2024-12-10T01:16:18,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742205_1381 (size=31255) 2024-12-10T01:16:18,381 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 301a23263bb8ba2547c849aa87975635#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:18,381 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/311d018b70374aae9967fc97a0fd4aee is 50, key is test_row_0/C:col10/1733793375473/Put/seqid=0 2024-12-10T01:16:18,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742206_1382 (size=12915) 2024-12-10T01:16:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. as already flushing 2024-12-10T01:16:18,493 INFO [Thread-1404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1404. 2024-12-10T01:16:18,493 DEBUG [Thread-1404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:18,493 DEBUG [Thread-1404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:18,493 DEBUG [Thread-1404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:18,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:18,778 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/58d8bd2ac56542a8a354ffbfc5b52590 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/58d8bd2ac56542a8a354ffbfc5b52590 2024-12-10T01:16:18,779 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=573, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/96195273ee54499da44e5067fbad2001 2024-12-10T01:16:18,783 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/A of 301a23263bb8ba2547c849aa87975635 into 58d8bd2ac56542a8a354ffbfc5b52590(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:18,783 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:18,783 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/A, priority=12, startTime=1733793377940; duration=0sec 2024-12-10T01:16:18,783 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:18,783 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:A 2024-12-10T01:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/dc8afdb028bd425e889b862592d9693f is 50, key is test_row_0/B:col10/1733793376320/Put/seqid=0 2024-12-10T01:16:18,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742207_1383 (size=12301) 2024-12-10T01:16:18,788 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/311d018b70374aae9967fc97a0fd4aee as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/311d018b70374aae9967fc97a0fd4aee 2024-12-10T01:16:18,791 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 301a23263bb8ba2547c849aa87975635/C of 301a23263bb8ba2547c849aa87975635 into 311d018b70374aae9967fc97a0fd4aee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:18,791 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:18,792 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., storeName=301a23263bb8ba2547c849aa87975635/C, priority=12, startTime=1733793377940; duration=0sec 2024-12-10T01:16:18,792 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:18,792 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 301a23263bb8ba2547c849aa87975635:C 2024-12-10T01:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:19,189 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/dc8afdb028bd425e889b862592d9693f 2024-12-10T01:16:19,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fcd3241a487d4f12805bc8a10e8c59ba is 50, key is test_row_0/C:col10/1733793376320/Put/seqid=0 2024-12-10T01:16:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742208_1384 (size=12301) 2024-12-10T01:16:19,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fcd3241a487d4f12805bc8a10e8c59ba 2024-12-10T01:16:19,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/96195273ee54499da44e5067fbad2001 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/96195273ee54499da44e5067fbad2001 2024-12-10T01:16:19,622 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/96195273ee54499da44e5067fbad2001, entries=150, sequenceid=573, filesize=30.5 K 2024-12-10T01:16:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/dc8afdb028bd425e889b862592d9693f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/dc8afdb028bd425e889b862592d9693f 2024-12-10T01:16:19,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/dc8afdb028bd425e889b862592d9693f, entries=150, sequenceid=573, filesize=12.0 K 2024-12-10T01:16:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fcd3241a487d4f12805bc8a10e8c59ba as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fcd3241a487d4f12805bc8a10e8c59ba 2024-12-10T01:16:19,630 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fcd3241a487d4f12805bc8a10e8c59ba, entries=150, sequenceid=573, filesize=12.0 K 2024-12-10T01:16:19,631 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 301a23263bb8ba2547c849aa87975635 in 1671ms, sequenceid=573, compaction requested=false 2024-12-10T01:16:19,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 301a23263bb8ba2547c849aa87975635: 2024-12-10T01:16:19,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:19,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-10T01:16:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-10T01:16:19,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-10T01:16:19,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7400 sec 2024-12-10T01:16:19,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7460 sec 2024-12-10T01:16:19,651 INFO [Thread-1408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1408. 2024-12-10T01:16:19,652 DEBUG [Thread-1408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:19,652 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,652 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:19,661 INFO [Thread-1402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1402. 2024-12-10T01:16:19,661 DEBUG [Thread-1402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:19,661 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,661 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:19,683 INFO [Thread-1406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1406. 2024-12-10T01:16:19,683 DEBUG [Thread-1406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:19,683 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,683 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:19,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:20,194 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-12-10T01:16:20,194 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:20,194 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:20,195 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:20,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:21,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:16:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T01:16:21,069 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1539 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4614 rows 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1543 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4622 rows 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1535 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4600 rows 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1533 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4595 rows 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1551 2024-12-10T01:16:21,070 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4644 rows 2024-12-10T01:16:21,070 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:16:21,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:16:21,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:21,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:21,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:21,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:16:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:21,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793381073"}]},"ts":"1733793381073"} 2024-12-10T01:16:21,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:16:21,074 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:16:21,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:16:21,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, UNASSIGN}] 2024-12-10T01:16:21,077 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, UNASSIGN 2024-12-10T01:16:21,078 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:21,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, UNASSIGN because future has completed 2024-12-10T01:16:21,080 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:16:21,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:21,233 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(122): Close 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:21,233 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:16:21,234 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1722): Closing 301a23263bb8ba2547c849aa87975635, disabling compactions & flushes 2024-12-10T01:16:21,234 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:21,234 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:21,234 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. after waiting 0 ms 2024-12-10T01:16:21,234 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:21,234 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(2902): Flushing 301a23263bb8ba2547c849aa87975635 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=A 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=B 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 301a23263bb8ba2547c849aa87975635, store=C 2024-12-10T01:16:21,235 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:21,245 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121049fe778f334441c9b221328e46cca1de_301a23263bb8ba2547c849aa87975635 is 50, key is test_row_0/A:col10/1733793379659/Put/seqid=0 2024-12-10T01:16:21,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742209_1385 (size=12454) 2024-12-10T01:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:21,650 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:21,659 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121049fe778f334441c9b221328e46cca1de_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121049fe778f334441c9b221328e46cca1de_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:21,660 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b6f7ecdf49a441a1957f7f0c374c782a, store: [table=TestAcidGuarantees family=A region=301a23263bb8ba2547c849aa87975635] 2024-12-10T01:16:21,661 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b6f7ecdf49a441a1957f7f0c374c782a is 175, key is test_row_0/A:col10/1733793379659/Put/seqid=0 2024-12-10T01:16:21,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742210_1386 (size=31255) 2024-12-10T01:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:22,065 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=584, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b6f7ecdf49a441a1957f7f0c374c782a 2024-12-10T01:16:22,073 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/0a68ac0beb71437f89c29de4fd6e6f8e is 50, key is test_row_0/B:col10/1733793379659/Put/seqid=0 2024-12-10T01:16:22,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742211_1387 (size=12301) 2024-12-10T01:16:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:22,479 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/0a68ac0beb71437f89c29de4fd6e6f8e 2024-12-10T01:16:22,491 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fd732b55d89c4ccd952a75fecfc16d91 is 50, key is test_row_0/C:col10/1733793379659/Put/seqid=0 2024-12-10T01:16:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742212_1388 (size=12301) 2024-12-10T01:16:22,896 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fd732b55d89c4ccd952a75fecfc16d91 2024-12-10T01:16:22,944 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/A/b6f7ecdf49a441a1957f7f0c374c782a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b6f7ecdf49a441a1957f7f0c374c782a 2024-12-10T01:16:22,948 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b6f7ecdf49a441a1957f7f0c374c782a, entries=150, sequenceid=584, filesize=30.5 K 2024-12-10T01:16:22,949 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/B/0a68ac0beb71437f89c29de4fd6e6f8e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0a68ac0beb71437f89c29de4fd6e6f8e 2024-12-10T01:16:22,951 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0a68ac0beb71437f89c29de4fd6e6f8e, entries=150, sequenceid=584, filesize=12.0 K 2024-12-10T01:16:22,952 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/.tmp/C/fd732b55d89c4ccd952a75fecfc16d91 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fd732b55d89c4ccd952a75fecfc16d91 2024-12-10T01:16:22,955 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fd732b55d89c4ccd952a75fecfc16d91, entries=150, sequenceid=584, filesize=12.0 K 2024-12-10T01:16:22,956 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 301a23263bb8ba2547c849aa87975635 in 1722ms, sequenceid=584, compaction requested=true 2024-12-10T01:16:22,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62] to archive 2024-12-10T01:16:22,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d9cd123f6f5d403281c9acc9d6b01e53 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/472853891f924f9e98861ed94cf4ac04 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/c8411a49d4db46aabc924125236cc0cf 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/521c17581d6f4f7ba571e151d0106eea 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/4c0287dee127481bacf9cf88d1a76fac 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/939948ff3bec44ef860f0b02a39864b8 2024-12-10T01:16:22,959 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/bdd61fc75fa24e1580c219bc38551684 2024-12-10T01:16:22,960 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/fdc5f197f3ed437986c9a84717adcb7e 2024-12-10T01:16:22,960 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/23862f753084443e9121c8e05bd271a1 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/79e5170c456e46d08fc324394dd99318 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/df3ab0b67b2345b6bb2b90f7bbed8ed6 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/406e307107e746a8963d1f6db2ad689c 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/a54b1d8c367e4b06b9282c01e9e0fd5a 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/6484994d853b451589ec39025e9dee4d 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/d3ee98e23f6646f2b8094732cc581cca 2024-12-10T01:16:22,961 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b017dffdc9bb47cd90982230feb5ab71 2024-12-10T01:16:22,962 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/86bb1a3f04a4433d9acfa46ef1cd8cbf 2024-12-10T01:16:22,962 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/74afc5e21ab34019ab189cfbe754cf62 2024-12-10T01:16:22,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/994edde70f484f67bf0ff56e9161c41f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0c8fffe3de564610907b740a051f18c0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/58ca659c6e5643fcb3c49007dc7da255, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/70c15e028dd74903bb1a1c981a9e98fc, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436] to archive 2024-12-10T01:16:22,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:22,965 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/43714a02554d41deb06e5de667f05e99 2024-12-10T01:16:22,965 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/5bac26f3c49e4df0aa34434a6a81e23e 2024-12-10T01:16:22,965 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/32dadf92881f498c825909a4bd019ee7 2024-12-10T01:16:22,966 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/994edde70f484f67bf0ff56e9161c41f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/994edde70f484f67bf0ff56e9161c41f 2024-12-10T01:16:22,966 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/22f6eae235c2460fbfb67a7b74ae6fc0 2024-12-10T01:16:22,966 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f550f82e76f540169fccf5735a400208 2024-12-10T01:16:22,966 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0c8fffe3de564610907b740a051f18c0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0c8fffe3de564610907b740a051f18c0 2024-12-10T01:16:22,966 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1694a86ca2f849969af3654441e203b8 2024-12-10T01:16:22,967 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/ff56256682c444c7a7150115c934161b 2024-12-10T01:16:22,967 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/58ca659c6e5643fcb3c49007dc7da255 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/58ca659c6e5643fcb3c49007dc7da255 2024-12-10T01:16:22,967 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/c1f31bd37e164f2a9a9b5f64abdb30f6 2024-12-10T01:16:22,967 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/70c15e028dd74903bb1a1c981a9e98fc to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/70c15e028dd74903bb1a1c981a9e98fc 2024-12-10T01:16:22,967 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/1b1ad5b8f1ef4a188b144f393ad1c632 2024-12-10T01:16:22,968 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/111a47ce3ca04e10bdd1c42fee9f3878 2024-12-10T01:16:22,968 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/3adab1591821468da130174b78ff1af9 2024-12-10T01:16:22,968 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/f6e542a98f254266bb2d0cd0ed84429f 2024-12-10T01:16:22,968 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/d6283354999142d19fdfc7f4b0891a06 2024-12-10T01:16:22,968 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/91bdd91a5f2847c190241f6d7afb2436 2024-12-10T01:16:22,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f0299dc421be4d43834bca83a7bf1dff, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4f5cf744802142fabec45a975d53316e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4b8980a7b6a3486ab92507f732cd418b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469] to archive 2024-12-10T01:16:22,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9f39f199b177420abfb73ac2d3ff36e8 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/da6b71906f4345bbba82875393d4f391 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f0299dc421be4d43834bca83a7bf1dff to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f0299dc421be4d43834bca83a7bf1dff 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/665c114e08dd4183a9d94590fc7fdfe1 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/b03d7391e1534fb2b19d8f6058406163 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/2e29b54d8bea4679a613a4b24970e3c0 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4f5cf744802142fabec45a975d53316e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4f5cf744802142fabec45a975d53316e 2024-12-10T01:16:22,972 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/be4586ca9a46415d99f77dd0cf045705 2024-12-10T01:16:22,973 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/3f35acc49ebd4fabaff96b28ce3e91e3 2024-12-10T01:16:22,973 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fabae456288d44d08a400a9b70ca68a2 2024-12-10T01:16:22,973 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/f5c8b3a3e4cf444a8b7ba6b72965ee8f 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ee81a4ef883343b29e9f4166fbb0cfc1 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/94081604126f4ddeb219a899ab36bce2 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9be567e4b2354ec88709f39c15c18cf9 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4b8980a7b6a3486ab92507f732cd418b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/4b8980a7b6a3486ab92507f732cd418b 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/ba72a5a560234862ba72ea2e6ba8087b 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/37aad8fe3ded4faface71acb6312f916 2024-12-10T01:16:22,974 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/9c8b4edc47064f0bbaca0be079169469 2024-12-10T01:16:22,978 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits/587.seqid, newMaxSeqId=587, maxSeqId=4 2024-12-10T01:16:22,978 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. 2024-12-10T01:16:22,978 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1676): Region close journal for 301a23263bb8ba2547c849aa87975635: Waiting for close lock at 1733793381233Running coprocessor pre-close hooks at 1733793381233Disabling compacts and flushes for region at 1733793381233Disabling writes for close at 1733793381234 (+1 ms)Obtaining lock to block concurrent updates at 1733793381234Preparing flush snapshotting stores in 301a23263bb8ba2547c849aa87975635 at 1733793381234Finished memstore snapshotting TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733793381235 (+1 ms)Flushing stores of TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635. at 1733793381236 (+1 ms)Flushing 301a23263bb8ba2547c849aa87975635/A: creating writer at 1733793381236Flushing mob file 301a23263bb8ba2547c849aa87975635/A: appending metadata at 1733793381245 (+9 ms)Flushing mob file 301a23263bb8ba2547c849aa87975635/A: closing flushed file at 1733793381245Flushing 301a23263bb8ba2547c849aa87975635/A: appending metadata at 1733793381660 (+415 ms)Flushing 301a23263bb8ba2547c849aa87975635/A: closing flushed file at 1733793381660Flushing 301a23263bb8ba2547c849aa87975635/B: creating writer at 1733793382070 (+410 ms)Flushing 301a23263bb8ba2547c849aa87975635/B: appending metadata at 1733793382073 (+3 ms)Flushing 301a23263bb8ba2547c849aa87975635/B: closing flushed file at 1733793382073Flushing 301a23263bb8ba2547c849aa87975635/C: creating writer at 1733793382488 (+415 ms)Flushing 301a23263bb8ba2547c849aa87975635/C: appending metadata at 1733793382491 (+3 ms)Flushing 301a23263bb8ba2547c849aa87975635/C: closing flushed file at 1733793382491Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46cdeb9d: reopening flushed file at 1733793382944 (+453 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a0882f: reopening flushed file at 1733793382948 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17043abf: reopening flushed file at 1733793382951 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 301a23263bb8ba2547c849aa87975635 in 1722ms, sequenceid=584, compaction requested=true at 1733793382956 (+5 ms)Writing region close event to WAL at 1733793382975 (+19 ms)Running coprocessor post-close hooks at 1733793382978 (+3 ms)Closed at 1733793382978 2024-12-10T01:16:22,980 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(157): Closed 301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:22,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=301a23263bb8ba2547c849aa87975635, regionState=CLOSED 2024-12-10T01:16:22,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:22,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T01:16:22,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseRegionProcedure 301a23263bb8ba2547c849aa87975635, server=fd1a2b66caf8,38925,1733793262612 in 1.9020 sec 2024-12-10T01:16:22,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-12-10T01:16:22,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=301a23263bb8ba2547c849aa87975635, UNASSIGN in 1.9070 sec 2024-12-10T01:16:22,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-10T01:16:22,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9100 sec 2024-12-10T01:16:22,987 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793382987"}]},"ts":"1733793382987"} 2024-12-10T01:16:22,988 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:16:22,988 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:16:22,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9180 sec 2024-12-10T01:16:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T01:16:23,221 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:23,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,227 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=101, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T01:16:23,228 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=101, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,231 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,235 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits] 2024-12-10T01:16:23,239 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/58d8bd2ac56542a8a354ffbfc5b52590 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/58d8bd2ac56542a8a354ffbfc5b52590 2024-12-10T01:16:23,239 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/96195273ee54499da44e5067fbad2001 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/96195273ee54499da44e5067fbad2001 2024-12-10T01:16:23,239 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b6f7ecdf49a441a1957f7f0c374c782a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/A/b6f7ecdf49a441a1957f7f0c374c782a 2024-12-10T01:16:23,243 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0a68ac0beb71437f89c29de4fd6e6f8e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/0a68ac0beb71437f89c29de4fd6e6f8e 2024-12-10T01:16:23,243 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/a2ea893c26b24e9781e82f923bd3f3ff to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/a2ea893c26b24e9781e82f923bd3f3ff 2024-12-10T01:16:23,244 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/dc8afdb028bd425e889b862592d9693f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/B/dc8afdb028bd425e889b862592d9693f 2024-12-10T01:16:23,248 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fcd3241a487d4f12805bc8a10e8c59ba to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fcd3241a487d4f12805bc8a10e8c59ba 2024-12-10T01:16:23,248 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/311d018b70374aae9967fc97a0fd4aee to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/311d018b70374aae9967fc97a0fd4aee 2024-12-10T01:16:23,248 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fd732b55d89c4ccd952a75fecfc16d91 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/C/fd732b55d89c4ccd952a75fecfc16d91 2024-12-10T01:16:23,252 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits/587.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635/recovered.edits/587.seqid 2024-12-10T01:16:23,253 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:16:23,253 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:16:23,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T01:16:23,264 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100ef95a80e6064809a84195f4d5de11fa_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100ef95a80e6064809a84195f4d5de11fa_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,264 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a057e7f71f34a57b26867eb154fafd3_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a057e7f71f34a57b26867eb154fafd3_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,264 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121049fe778f334441c9b221328e46cca1de_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121049fe778f334441c9b221328e46cca1de_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,264 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121067c98513814a4d3a811b5035819e6003_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121067c98513814a4d3a811b5035819e6003_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,265 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f0d288d56bb406fb3d8a549a79a1d64_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f0d288d56bb406fb3d8a549a79a1d64_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,265 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a170ff6c2b841a4927790ff50631bd8_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a170ff6c2b841a4927790ff50631bd8_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,265 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121086abe3bf5cfb4e7b90fc223f8f090a44_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121086abe3bf5cfb4e7b90fc223f8f090a44_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,265 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b57b4e98e42479282543110d2b320c1_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b57b4e98e42479282543110d2b320c1_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,266 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210940e66f265ba4b50a45389a90b1dee21_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210940e66f265ba4b50a45389a90b1dee21_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,266 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bdac80b061a54310bc0de119aa9605ac_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bdac80b061a54310bc0de119aa9605ac_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,266 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b140bef0eed74740b7b6677d4d084459_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b140bef0eed74740b7b6677d4d084459_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,267 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ca25eff7dc594d6e9006982794f6c6d9_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ca25eff7dc594d6e9006982794f6c6d9_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,267 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d541c9a20fa84e3c8ba03c7e4a5a0aab_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d541c9a20fa84e3c8ba03c7e4a5a0aab_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,267 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc4565bc3b3f4fd483b23a8ba9b15959_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc4565bc3b3f4fd483b23a8ba9b15959_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,267 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e857575b29a3428490c660067d2f6a96_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e857575b29a3428490c660067d2f6a96_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,267 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210defe99a2d6b04f74aa67e91993fd816f_301a23263bb8ba2547c849aa87975635 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210defe99a2d6b04f74aa67e91993fd816f_301a23263bb8ba2547c849aa87975635 2024-12-10T01:16:23,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:16:23,270 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=101, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,272 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:16:23,275 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:16:23,276 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=101, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,276 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:16:23,276 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793383276"}]},"ts":"9223372036854775807"} 2024-12-10T01:16:23,278 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:16:23,278 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 301a23263bb8ba2547c849aa87975635, NAME => 'TestAcidGuarantees,,1733793354670.301a23263bb8ba2547c849aa87975635.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:16:23,278 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:16:23,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793383278"}]},"ts":"9223372036854775807"} 2024-12-10T01:16:23,281 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:16:23,282 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=101, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-12-10T01:16:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T01:16:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:16:23,344 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:23,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=257 (was 256) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 454), ProcessCount=11 (was 11), AvailableMemoryMB=7381 (was 7573) 2024-12-10T01:16:23,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=257, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=7381 2024-12-10T01:16:23,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:16:23,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:16:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:23,366 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:16:23,366 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:23,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 102 2024-12-10T01:16:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:23,367 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:16:23,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742213_1389 (size=1061) 2024-12-10T01:16:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:23,778 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:16:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742214_1390 (size=53) 2024-12-10T01:16:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:24,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:16:24,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7be4f2ce41631b00d075356558ca2ac, disabling compactions & flushes 2024-12-10T01:16:24,187 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,188 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,188 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. after waiting 0 ms 2024-12-10T01:16:24,188 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,188 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,188 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7be4f2ce41631b00d075356558ca2ac: Waiting for close lock at 1733793384187Disabling compacts and flushes for region at 1733793384187Disabling writes for close at 1733793384188 (+1 ms)Writing region close event to WAL at 1733793384188Closed at 1733793384188 2024-12-10T01:16:24,190 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:16:24,191 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793384190"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793384190"}]},"ts":"1733793384190"} 2024-12-10T01:16:24,194 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:16:24,197 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:16:24,197 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793384197"}]},"ts":"1733793384197"} 2024-12-10T01:16:24,199 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:16:24,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, ASSIGN}] 2024-12-10T01:16:24,201 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, ASSIGN 2024-12-10T01:16:24,201 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:16:24,353 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=f7be4f2ce41631b00d075356558ca2ac, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:24,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, ASSIGN because future has completed 2024-12-10T01:16:24,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:24,516 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,516 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:16:24,517 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,518 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:16:24,518 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,518 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,519 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,521 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:24,521 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7be4f2ce41631b00d075356558ca2ac columnFamilyName A 2024-12-10T01:16:24,521 DEBUG [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:24,522 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(327): Store=f7be4f2ce41631b00d075356558ca2ac/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:24,522 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,523 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:24,523 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7be4f2ce41631b00d075356558ca2ac columnFamilyName B 2024-12-10T01:16:24,524 DEBUG [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:24,524 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(327): Store=f7be4f2ce41631b00d075356558ca2ac/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:24,524 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,526 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:24,526 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7be4f2ce41631b00d075356558ca2ac columnFamilyName C 2024-12-10T01:16:24,526 DEBUG [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:24,526 INFO [StoreOpener-f7be4f2ce41631b00d075356558ca2ac-1 {}] regionserver.HStore(327): Store=f7be4f2ce41631b00d075356558ca2ac/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:24,527 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,527 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,528 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,528 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,530 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,530 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,531 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:16:24,533 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,536 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:16:24,536 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened f7be4f2ce41631b00d075356558ca2ac; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72236965, jitterRate=0.07641465961933136}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:16:24,536 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:24,537 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for f7be4f2ce41631b00d075356558ca2ac: Running coprocessor pre-open hook at 1733793384518Writing region info on filesystem at 1733793384518Initializing all the Stores at 1733793384519 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793384519Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793384519Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793384519Cleaning up temporary data from old regions at 1733793384530 (+11 ms)Running coprocessor post-open hooks at 1733793384536 (+6 ms)Region opened successfully at 1733793384537 (+1 ms) 2024-12-10T01:16:24,539 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., pid=104, masterSystemTime=1733793384511 2024-12-10T01:16:24,541 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,541 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:24,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=f7be4f2ce41631b00d075356558ca2ac, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:24,544 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37249 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fd1a2b66caf8,38925,1733793262612, table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T01:16:24,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:24,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T01:16:24,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; OpenRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 in 189 msec 2024-12-10T01:16:24,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-10T01:16:24,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, ASSIGN in 349 msec 2024-12-10T01:16:24,552 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:16:24,552 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793384552"}]},"ts":"1733793384552"} 2024-12-10T01:16:24,553 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:16:24,554 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=102, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:16:24,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-12-10T01:16:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-10T01:16:25,520 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:25,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f130865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,523 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342b5aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46784bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,527 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,531 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38ce38e3 2024-12-10T01:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:16:25,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:16:25,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c37e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,536 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eee1366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2feaa2ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b620c86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414eec6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,541 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e4eae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2396f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,544 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f71899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b09f312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386cc551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,547 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24285787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e415759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebbc0f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,550 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3026f53f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fee11de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c62f443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371a5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5770ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c83dd13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5270f7ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2651fb8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e86fc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,561 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d307dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5ec752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5ce460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,564 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176acb64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,565 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7041b9fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ffe165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:25,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:25,567 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b84edbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:25,568 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68dfc7d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:25,569 DEBUG [Thread-1791 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,569 DEBUG [Thread-1789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,569 DEBUG [Thread-1778 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,569 DEBUG [Thread-1782 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,569 DEBUG [Thread-1785 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,570 DEBUG [Thread-1793 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,571 DEBUG [Thread-1774 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,571 DEBUG [Thread-1787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,572 DEBUG [Thread-1780 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,572 DEBUG [Thread-1776 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:25,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,574 DEBUG [Thread-1776 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,574 DEBUG [Thread-1776 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-10T01:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:25,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,577 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2] 2024-12-10T01:16:25,578 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:25,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:25,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:25,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/306fa0b45dcc4cc499c79f07c5f82e03 is 50, key is test_row_0/A:col10/1733793385579/Put/seqid=0 2024-12-10T01:16:25,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742215_1391 (size=12001) 2024-12-10T01:16:25,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793445602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793445609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733793385599, max timestamp=1733793385609Num uniques 859; 2024-12-10T01:16:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:25,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793385599, max timestamp=1733793385609Num uniques 859; 2024-12-10T01:16:25,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:25,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733793385599, max timestamp=1733793385609Num uniques 859; 2024-12-10T01:16:25,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T01:16:25,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793445643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793445646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793445654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:25,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793445719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793445719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:25,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793445759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793445759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793445769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:25,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:25,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793445930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793445930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793445969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793445969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793445980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/306fa0b45dcc4cc499c79f07c5f82e03 2024-12-10T01:16:26,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f39bd47df30342b8a4bb7dd733d07ba3 is 50, key is test_row_0/B:col10/1733793385579/Put/seqid=0 2024-12-10T01:16:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742216_1392 (size=12001) 2024-12-10T01:16:26,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f39bd47df30342b8a4bb7dd733d07ba3 2024-12-10T01:16:26,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/350aafc3a49f40b5ae7c289874c5ecf0 is 50, key is test_row_0/C:col10/1733793385579/Put/seqid=0 2024-12-10T01:16:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742217_1393 (size=12001) 2024-12-10T01:16:26,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793446239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793446239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793446279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793446280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793446289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:26,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/350aafc3a49f40b5ae7c289874c5ecf0 2024-12-10T01:16:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/306fa0b45dcc4cc499c79f07c5f82e03 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03 2024-12-10T01:16:26,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T01:16:26,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f39bd47df30342b8a4bb7dd733d07ba3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3 2024-12-10T01:16:26,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T01:16:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/350aafc3a49f40b5ae7c289874c5ecf0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0 2024-12-10T01:16:26,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T01:16:26,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f7be4f2ce41631b00d075356558ca2ac in 888ms, sequenceid=12, compaction requested=false 2024-12-10T01:16:26,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:26,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T01:16:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:26,496 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:26,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/80f135da44fc449c987a7f7b57987773 is 50, key is test_row_0/A:col10/1733793385653/Put/seqid=0 2024-12-10T01:16:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742218_1394 (size=12001) 2024-12-10T01:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793446771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793446772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793446790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793446799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793446799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793446889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793446889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:26,903 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/80f135da44fc449c987a7f7b57987773 2024-12-10T01:16:26,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/418df6de2960410fadbb53324f0453ac is 50, key is test_row_0/B:col10/1733793385653/Put/seqid=0 2024-12-10T01:16:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742219_1395 (size=12001) 2024-12-10T01:16:27,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793447100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793447100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/418df6de2960410fadbb53324f0453ac 2024-12-10T01:16:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/eca253fedaa841fdbfea1af742a843c4 is 50, key is test_row_0/C:col10/1733793385653/Put/seqid=0 2024-12-10T01:16:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742220_1396 (size=12001) 2024-12-10T01:16:27,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793447409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793447409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:27,721 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/eca253fedaa841fdbfea1af742a843c4 2024-12-10T01:16:27,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/80f135da44fc449c987a7f7b57987773 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773 2024-12-10T01:16:27,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T01:16:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/418df6de2960410fadbb53324f0453ac as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac 2024-12-10T01:16:27,731 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T01:16:27,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/eca253fedaa841fdbfea1af742a843c4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4 2024-12-10T01:16:27,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T01:16:27,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for f7be4f2ce41631b00d075356558ca2ac in 1241ms, sequenceid=46, compaction requested=false 2024-12-10T01:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-10T01:16:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-10T01:16:27,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-10T01:16:27,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-12-10T01:16:27,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1680 sec 2024-12-10T01:16:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:27,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:27,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:27,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/a4c757cd9e504f51a0d33e1d5347246d is 50, key is test_row_0/A:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742221_1397 (size=12001) 2024-12-10T01:16:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793447856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793447857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793447858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793447919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793447919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793447969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793447969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793447969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793448180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793448180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793448180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/a4c757cd9e504f51a0d33e1d5347246d 2024-12-10T01:16:28,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d9672e94448e42cc8cb17474f8a796dd is 50, key is test_row_0/B:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742222_1398 (size=12001) 2024-12-10T01:16:28,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793448489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793448489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793448489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,499 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:16:28,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d9672e94448e42cc8cb17474f8a796dd 2024-12-10T01:16:28,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/26c7bf362fef4b37be08b138fc7f0934 is 50, key is test_row_0/C:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:28,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742223_1399 (size=12001) 2024-12-10T01:16:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793448930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,935 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:28,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793448939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:28,944 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:29,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793448999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793449000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793449010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:29,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/26c7bf362fef4b37be08b138fc7f0934 2024-12-10T01:16:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/a4c757cd9e504f51a0d33e1d5347246d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d 2024-12-10T01:16:29,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d9672e94448e42cc8cb17474f8a796dd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd 2024-12-10T01:16:29,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:16:29,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/26c7bf362fef4b37be08b138fc7f0934 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934 2024-12-10T01:16:29,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934, entries=150, sequenceid=58, filesize=11.7 K 2024-12-10T01:16:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f7be4f2ce41631b00d075356558ca2ac in 1301ms, sequenceid=58, compaction requested=true 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:29,115 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:29,115 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:29,115 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:29,115 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:29,116 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:29,116 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:29,116 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.2 K 2024-12-10T01:16:29,116 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.2 K 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f39bd47df30342b8a4bb7dd733d07ba3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733793385579 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 306fa0b45dcc4cc499c79f07c5f82e03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733793385579 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f135da44fc449c987a7f7b57987773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733793385627 2024-12-10T01:16:29,116 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 418df6de2960410fadbb53324f0453ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733793385627 2024-12-10T01:16:29,117 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d9672e94448e42cc8cb17474f8a796dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:29,117 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c757cd9e504f51a0d33e1d5347246d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:29,122 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:29,122 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/8c5ab5919674497db2a9a2674cb82f51 is 50, key is test_row_0/A:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:29,123 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:29,123 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/bfe28543486b4d49a5cee2402dbec7b5 is 50, key is test_row_0/B:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:29,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742224_1400 (size=12104) 2024-12-10T01:16:29,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742225_1401 (size=12104) 2024-12-10T01:16:29,132 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/bfe28543486b4d49a5cee2402dbec7b5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/bfe28543486b4d49a5cee2402dbec7b5 2024-12-10T01:16:29,135 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into bfe28543486b4d49a5cee2402dbec7b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:29,135 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:29,135 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793389115; duration=0sec 2024-12-10T01:16:29,135 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:29,136 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:29,136 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:29,136 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:29,136 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:29,136 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:29,136 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.2 K 2024-12-10T01:16:29,137 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 350aafc3a49f40b5ae7c289874c5ecf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733793385579 2024-12-10T01:16:29,137 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting eca253fedaa841fdbfea1af742a843c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733793385627 2024-12-10T01:16:29,137 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c7bf362fef4b37be08b138fc7f0934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:29,150 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:29,151 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/e3743006362346d0bbab48b5b3af6400 is 50, key is test_row_0/C:col10/1733793387812/Put/seqid=0 2024-12-10T01:16:29,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742226_1402 (size=12104) 2024-12-10T01:16:29,530 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/8c5ab5919674497db2a9a2674cb82f51 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/8c5ab5919674497db2a9a2674cb82f51 2024-12-10T01:16:29,534 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into 8c5ab5919674497db2a9a2674cb82f51(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:29,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:29,535 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793389115; duration=0sec 2024-12-10T01:16:29,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:29,535 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:29,558 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/e3743006362346d0bbab48b5b3af6400 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e3743006362346d0bbab48b5b3af6400 2024-12-10T01:16:29,563 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into e3743006362346d0bbab48b5b3af6400(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:29,563 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:29,563 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793389115; duration=0sec 2024-12-10T01:16:29,563 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:29,563 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T01:16:29,739 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:29,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-10T01:16:29,742 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:29,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:29,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T01:16:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T01:16:29,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T01:16:29,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:29,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/2c31ffc1fa7a4feb8034b55ad8d394e1 is 50, key is test_row_0/A:col10/1733793387836/Put/seqid=0 2024-12-10T01:16:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742227_1403 (size=12001) 2024-12-10T01:16:29,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/2c31ffc1fa7a4feb8034b55ad8d394e1 2024-12-10T01:16:29,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7b1989ea096742189d82858217cba2f1 is 50, key is test_row_0/B:col10/1733793387836/Put/seqid=0 2024-12-10T01:16:29,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742228_1404 (size=12001) 2024-12-10T01:16:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:30,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793450036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793450042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T01:16:30,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793450149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793450149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793450160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7b1989ea096742189d82858217cba2f1 2024-12-10T01:16:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d68caaf8088d4ec88e7f492ed8c2e799 is 50, key is test_row_0/C:col10/1733793387836/Put/seqid=0 2024-12-10T01:16:30,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742229_1405 (size=12001) 2024-12-10T01:16:30,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793450359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793450359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T01:16:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793450370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793450669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793450669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793450689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d68caaf8088d4ec88e7f492ed8c2e799 2024-12-10T01:16:30,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/2c31ffc1fa7a4feb8034b55ad8d394e1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1 2024-12-10T01:16:30,726 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T01:16:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7b1989ea096742189d82858217cba2f1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1 2024-12-10T01:16:30,730 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T01:16:30,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d68caaf8088d4ec88e7f492ed8c2e799 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799 2024-12-10T01:16:30,733 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T01:16:30,734 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f7be4f2ce41631b00d075356558ca2ac in 838ms, sequenceid=81, compaction requested=false 2024-12-10T01:16:30,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:30,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:30,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-10T01:16:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-10T01:16:30,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-10T01:16:30,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 992 msec 2024-12-10T01:16:30,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 997 msec 2024-12-10T01:16:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T01:16:30,879 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:30,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-10T01:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:30,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:30,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:30,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:30,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:16:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:30,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:30,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:30,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/29cfbeb8461c46b583a2291f570c3461 is 50, key is test_row_0/A:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742230_1406 (size=14341) 2024-12-10T01:16:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793450984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793450987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793451099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793451099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793451179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793451179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793451199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793451309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793451310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/29cfbeb8461c46b583a2291f570c3461 2024-12-10T01:16:31,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/19dc0f574c514c55bb2f9849a562fcb6 is 50, key is test_row_0/B:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:31,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742231_1407 (size=12001) 2024-12-10T01:16:31,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793451619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793451619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:31,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/19dc0f574c514c55bb2f9849a562fcb6 2024-12-10T01:16:31,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/8e874e1cc2c34395a0ad82c873825a68 is 50, key is test_row_0/C:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:31,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742232_1408 (size=12001) 2024-12-10T01:16:31,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:31,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:32,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793452129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793452129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/8e874e1cc2c34395a0ad82c873825a68 2024-12-10T01:16:32,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/29cfbeb8461c46b583a2291f570c3461 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461 2024-12-10T01:16:32,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461, entries=200, sequenceid=94, filesize=14.0 K 2024-12-10T01:16:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/19dc0f574c514c55bb2f9849a562fcb6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6 2024-12-10T01:16:32,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6, entries=150, sequenceid=94, filesize=11.7 K 2024-12-10T01:16:32,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/8e874e1cc2c34395a0ad82c873825a68 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68 2024-12-10T01:16:32,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68, entries=150, sequenceid=94, filesize=11.7 K 2024-12-10T01:16:32,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f7be4f2ce41631b00d075356558ca2ac in 1243ms, sequenceid=94, compaction requested=true 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:32,186 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:32,186 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:32,190 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:32,190 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:32,190 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:32,190 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:32,190 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,190 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,190 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/8c5ab5919674497db2a9a2674cb82f51, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=37.5 K 2024-12-10T01:16:32,190 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/bfe28543486b4d49a5cee2402dbec7b5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.3 K 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5ab5919674497db2a9a2674cb82f51, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe28543486b4d49a5cee2402dbec7b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1989ea096742189d82858217cba2f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733793387836 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c31ffc1fa7a4feb8034b55ad8d394e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733793387836 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29cfbeb8461c46b583a2291f570c3461, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:32,191 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 19dc0f574c514c55bb2f9849a562fcb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:32,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:32,197 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:32,198 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/25ac9922a3194070829e7dc4e92a4955 is 50, key is test_row_0/B:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:32,198 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:32,199 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/a1f91d13303b4a70a39b4b72aaf89f6f is 50, key is test_row_0/A:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:32,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/f84732cd60054025acc237488fb47aa0 is 50, key is test_row_0/A:col10/1733793392194/Put/seqid=0 2024-12-10T01:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742234_1410 (size=12207) 2024-12-10T01:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742235_1411 (size=14341) 2024-12-10T01:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742233_1409 (size=12207) 2024-12-10T01:16:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793452221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793452221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793452222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793452329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793452339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793452339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793452540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793452549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793452549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/f84732cd60054025acc237488fb47aa0 2024-12-10T01:16:32,613 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/25ac9922a3194070829e7dc4e92a4955 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/25ac9922a3194070829e7dc4e92a4955 2024-12-10T01:16:32,623 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/a1f91d13303b4a70a39b4b72aaf89f6f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a1f91d13303b4a70a39b4b72aaf89f6f 2024-12-10T01:16:32,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7c0e30f0388747f8a1842c6082e50944 is 50, key is test_row_0/B:col10/1733793392194/Put/seqid=0 2024-12-10T01:16:32,627 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into 25ac9922a3194070829e7dc4e92a4955(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:32,627 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into a1f91d13303b4a70a39b4b72aaf89f6f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:32,627 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793392186; duration=0sec 2024-12-10T01:16:32,627 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793392186; duration=0sec 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:32,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742236_1412 (size=12001) 2024-12-10T01:16:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:32,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e3743006362346d0bbab48b5b3af6400, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.3 K 2024-12-10T01:16:32,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e3743006362346d0bbab48b5b3af6400, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733793386759 2024-12-10T01:16:32,629 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d68caaf8088d4ec88e7f492ed8c2e799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733793387836 2024-12-10T01:16:32,629 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e874e1cc2c34395a0ad82c873825a68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:32,633 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:32,634 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/74f43a9dd7d840c8bb4a746d355fca80 is 50, key is test_row_0/C:col10/1733793390942/Put/seqid=0 2024-12-10T01:16:32,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742237_1413 (size=12207) 2024-12-10T01:16:32,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793452849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793452859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793452859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:32,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:32,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7c0e30f0388747f8a1842c6082e50944 2024-12-10T01:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:33,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/57c416c7400b4822be7d805e836c3229 is 50, key is test_row_0/C:col10/1733793392194/Put/seqid=0 2024-12-10T01:16:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742238_1414 (size=12001) 2024-12-10T01:16:33,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/57c416c7400b4822be7d805e836c3229 2024-12-10T01:16:33,041 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/74f43a9dd7d840c8bb4a746d355fca80 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/74f43a9dd7d840c8bb4a746d355fca80 2024-12-10T01:16:33,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/f84732cd60054025acc237488fb47aa0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0 2024-12-10T01:16:33,045 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into 74f43a9dd7d840c8bb4a746d355fca80(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:33,045 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:33,045 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793392186; duration=0sec 2024-12-10T01:16:33,045 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:33,045 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:33,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0, entries=200, sequenceid=114, filesize=14.0 K 2024-12-10T01:16:33,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/7c0e30f0388747f8a1842c6082e50944 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944 2024-12-10T01:16:33,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T01:16:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/57c416c7400b4822be7d805e836c3229 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229 2024-12-10T01:16:33,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T01:16:33,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f7be4f2ce41631b00d075356558ca2ac in 862ms, sequenceid=114, compaction requested=false 2024-12-10T01:16:33,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:33,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:33,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/d42c6ab292b14a48969e0093a81083a1 is 50, key is test_row_0/A:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:33,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742239_1415 (size=14441) 2024-12-10T01:16:33,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793453203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793453209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793453320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793453320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793453359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793453369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793453370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793453530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793453530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/d42c6ab292b14a48969e0093a81083a1 2024-12-10T01:16:33,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/61b71dc3e6414cd19133f6db1c61e764 is 50, key is test_row_0/B:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:33,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742240_1416 (size=12101) 2024-12-10T01:16:33,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793453839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793453839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:33,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:33,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/61b71dc3e6414cd19133f6db1c61e764 2024-12-10T01:16:33,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/e0702c695db34a26bc1b8464dc0efe9e is 50, key is test_row_0/C:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:33,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742241_1417 (size=12101) 2024-12-10T01:16:33,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:33,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:34,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:34,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793454349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793454349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:34,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/e0702c695db34a26bc1b8464dc0efe9e 2024-12-10T01:16:34,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/d42c6ab292b14a48969e0093a81083a1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1 2024-12-10T01:16:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793454379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:34,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793454379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:34,387 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:34,387 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793454390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:34,394 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:34,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1, entries=200, sequenceid=131, filesize=14.1 K 2024-12-10T01:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:34,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/61b71dc3e6414cd19133f6db1c61e764 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764 2024-12-10T01:16:34,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T01:16:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/e0702c695db34a26bc1b8464dc0efe9e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e 2024-12-10T01:16:34,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T01:16:34,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f7be4f2ce41631b00d075356558ca2ac in 1247ms, sequenceid=131, compaction requested=true 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:34,402 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:34,402 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:34,403 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:34,403 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:34,403 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:34,403 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:34,403 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,403 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,403 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a1f91d13303b4a70a39b4b72aaf89f6f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=40.0 K 2024-12-10T01:16:34,403 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/25ac9922a3194070829e7dc4e92a4955, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.5 K 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ac9922a3194070829e7dc4e92a4955, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f91d13303b4a70a39b4b72aaf89f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84732cd60054025acc237488fb47aa0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733793390977 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0e30f0388747f8a1842c6082e50944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733793390977 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b71dc3e6414cd19133f6db1c61e764, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392221 2024-12-10T01:16:34,404 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42c6ab292b14a48969e0093a81083a1, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392210 2024-12-10T01:16:34,409 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#351 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:34,410 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/8a129b5388a34b728e7c721003a9df2b is 50, key is test_row_0/B:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:34,410 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:34,411 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/e56113fb0a654f15bca03e8bd6c8d2a4 is 50, key is test_row_0/A:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:34,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742242_1418 (size=12409) 2024-12-10T01:16:34,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742243_1419 (size=12409) 2024-12-10T01:16:34,418 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/8a129b5388a34b728e7c721003a9df2b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/8a129b5388a34b728e7c721003a9df2b 2024-12-10T01:16:34,421 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into 8a129b5388a34b728e7c721003a9df2b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:34,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:34,421 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793394402; duration=0sec 2024-12-10T01:16:34,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:34,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:34,421 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:34,422 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:34,422 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:34,422 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,422 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/74f43a9dd7d840c8bb4a746d355fca80, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.5 K 2024-12-10T01:16:34,422 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f43a9dd7d840c8bb4a746d355fca80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733793390031 2024-12-10T01:16:34,423 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c416c7400b4822be7d805e836c3229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733793390977 2024-12-10T01:16:34,423 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e0702c695db34a26bc1b8464dc0efe9e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392221 2024-12-10T01:16:34,429 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:34,430 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/35c8f6273d6246c3bce678048a931a8c is 50, key is test_row_0/C:col10/1733793393154/Put/seqid=0 2024-12-10T01:16:34,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742244_1420 (size=12409) 2024-12-10T01:16:34,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T01:16:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:34,440 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T01:16:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:34,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:34,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:34,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/763ab15837504bf0abc31bb73f3c768b is 50, key is test_row_0/A:col10/1733793393208/Put/seqid=0 2024-12-10T01:16:34,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742245_1421 (size=12151) 2024-12-10T01:16:34,822 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/e56113fb0a654f15bca03e8bd6c8d2a4 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/e56113fb0a654f15bca03e8bd6c8d2a4 2024-12-10T01:16:34,825 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into e56113fb0a654f15bca03e8bd6c8d2a4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:34,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:34,825 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793394402; duration=0sec 2024-12-10T01:16:34,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:34,825 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:34,836 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/35c8f6273d6246c3bce678048a931a8c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/35c8f6273d6246c3bce678048a931a8c 2024-12-10T01:16:34,840 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into 35c8f6273d6246c3bce678048a931a8c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:34,840 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:34,840 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793394402; duration=0sec 2024-12-10T01:16:34,840 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:34,840 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:34,850 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/763ab15837504bf0abc31bb73f3c768b 2024-12-10T01:16:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/11679408809a41269a5eee771036f7b1 is 50, key is test_row_0/B:col10/1733793393208/Put/seqid=0 2024-12-10T01:16:34,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742246_1422 (size=12151) 2024-12-10T01:16:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:35,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/11679408809a41269a5eee771036f7b1 2024-12-10T01:16:35,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/9bdb7bae50f2461898cae5bdc53760cf is 50, key is test_row_0/C:col10/1733793393208/Put/seqid=0 2024-12-10T01:16:35,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742247_1423 (size=12151) 2024-12-10T01:16:35,270 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/9bdb7bae50f2461898cae5bdc53760cf 2024-12-10T01:16:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/763ab15837504bf0abc31bb73f3c768b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b 2024-12-10T01:16:35,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b, entries=150, sequenceid=150, filesize=11.9 K 2024-12-10T01:16:35,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/11679408809a41269a5eee771036f7b1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1 2024-12-10T01:16:35,280 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1, entries=150, sequenceid=150, filesize=11.9 K 2024-12-10T01:16:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/9bdb7bae50f2461898cae5bdc53760cf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf 2024-12-10T01:16:35,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf, entries=150, sequenceid=150, filesize=11.9 K 2024-12-10T01:16:35,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for f7be4f2ce41631b00d075356558ca2ac in 844ms, sequenceid=150, compaction requested=false 2024-12-10T01:16:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-10T01:16:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-10T01:16:35,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-10T01:16:35,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4050 sec 2024-12-10T01:16:35,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 4.4100 sec 2024-12-10T01:16:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:35,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:16:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:35,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/522ebfee3539445e8791bdd09c2d5b2a is 50, key is test_row_0/A:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:35,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742248_1424 (size=12147) 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793395405, max timestamp=1733793395415Num uniques 859; 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793395405, max timestamp=1733793395415Num uniques 859; 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793395405, max timestamp=1733793395415Num uniques 859; 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:35,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793455460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793455461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:35,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793455579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793455579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:35,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/522ebfee3539445e8791bdd09c2d5b2a 2024-12-10T01:16:35,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/9d4986c0fdbd465e8393b468dd801617 is 50, key is test_row_0/B:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742249_1425 (size=9757) 2024-12-10T01:16:35,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793455789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793455789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793456100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793456100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/9d4986c0fdbd465e8393b468dd801617 2024-12-10T01:16:36,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/67cd7580278e46b0b9dd437443dbe521 is 50, key is test_row_0/C:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:36,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742250_1426 (size=9757) 2024-12-10T01:16:36,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793456399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793456399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,401 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:36,401 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793456419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,425 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/67cd7580278e46b0b9dd437443dbe521 2024-12-10T01:16:36,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/522ebfee3539445e8791bdd09c2d5b2a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a 2024-12-10T01:16:36,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a, entries=150, sequenceid=162, filesize=11.9 K 2024-12-10T01:16:36,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/9d4986c0fdbd465e8393b468dd801617 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617 2024-12-10T01:16:36,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617, entries=100, sequenceid=162, filesize=9.5 K 2024-12-10T01:16:36,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/67cd7580278e46b0b9dd437443dbe521 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521 2024-12-10T01:16:36,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521, entries=100, sequenceid=162, filesize=9.5 K 2024-12-10T01:16:36,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f7be4f2ce41631b00d075356558ca2ac in 1240ms, sequenceid=162, compaction requested=true 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:36,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:36,614 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:36,615 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:36,615 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:36,615 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/e56113fb0a654f15bca03e8bd6c8d2a4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=35.8 K 2024-12-10T01:16:36,615 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/8a129b5388a34b728e7c721003a9df2b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=33.5 K 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a129b5388a34b728e7c721003a9df2b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392221 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e56113fb0a654f15bca03e8bd6c8d2a4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392221 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 11679408809a41269a5eee771036f7b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733793393197 2024-12-10T01:16:36,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763ab15837504bf0abc31bb73f3c768b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733793393197 2024-12-10T01:16:36,616 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4986c0fdbd465e8393b468dd801617, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793395370 2024-12-10T01:16:36,616 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 522ebfee3539445e8791bdd09c2d5b2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793395367 2024-12-10T01:16:36,623 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:36,623 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d851fbfd14f4475588dc0e1adfd38dcd is 50, key is test_row_0/B:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:36,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T01:16:36,625 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:36,626 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/56fe40d051a646fbaec83e10155edd66 is 50, key is test_row_0/A:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T01:16:36,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742251_1427 (size=12561) 2024-12-10T01:16:36,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4cded33fdab74febacc593f515c77bef is 50, key is test_row_0/A:col10/1733793396623/Put/seqid=0 2024-12-10T01:16:36,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742253_1429 (size=16931) 2024-12-10T01:16:36,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4cded33fdab74febacc593f515c77bef 2024-12-10T01:16:36,636 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/56fe40d051a646fbaec83e10155edd66 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/56fe40d051a646fbaec83e10155edd66 2024-12-10T01:16:36,640 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into 56fe40d051a646fbaec83e10155edd66(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:36,640 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:36,640 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793396614; duration=0sec 2024-12-10T01:16:36,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:36,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:36,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:36,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:36,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:36,641 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:36,641 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/35c8f6273d6246c3bce678048a931a8c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=33.5 K 2024-12-10T01:16:36,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d69f6f27a4134ead8097a67462e78b17 is 50, key is test_row_0/B:col10/1733793396623/Put/seqid=0 2024-12-10T01:16:36,642 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c8f6273d6246c3bce678048a931a8c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733793392221 2024-12-10T01:16:36,642 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bdb7bae50f2461898cae5bdc53760cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733793393197 2024-12-10T01:16:36,642 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cd7580278e46b0b9dd437443dbe521, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793395370 2024-12-10T01:16:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742252_1428 (size=12561) 2024-12-10T01:16:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742254_1430 (size=12151) 2024-12-10T01:16:36,651 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:36,652 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/3f580d36be164aee9b205dd09aafabaa is 50, key is test_row_0/C:col10/1733793395370/Put/seqid=0 2024-12-10T01:16:36,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742255_1431 (size=12561) 2024-12-10T01:16:36,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793456646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793456648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793456759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793456759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793456979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793456979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,047 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d851fbfd14f4475588dc0e1adfd38dcd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d851fbfd14f4475588dc0e1adfd38dcd 2024-12-10T01:16:37,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d69f6f27a4134ead8097a67462e78b17 2024-12-10T01:16:37,050 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into d851fbfd14f4475588dc0e1adfd38dcd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:37,050 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:37,050 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793396614; duration=0sec 2024-12-10T01:16:37,050 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:37,050 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:37,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/36762bf1f754467d9a4f32d60fb1cb36 is 50, key is test_row_0/C:col10/1733793396623/Put/seqid=0 2024-12-10T01:16:37,059 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/3f580d36be164aee9b205dd09aafabaa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/3f580d36be164aee9b205dd09aafabaa 2024-12-10T01:16:37,062 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into 3f580d36be164aee9b205dd09aafabaa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:37,062 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:37,062 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793396614; duration=0sec 2024-12-10T01:16:37,062 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:37,062 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:37,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742256_1432 (size=12151) 2024-12-10T01:16:37,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/36762bf1f754467d9a4f32d60fb1cb36 2024-12-10T01:16:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4cded33fdab74febacc593f515c77bef as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef 2024-12-10T01:16:37,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef, entries=250, sequenceid=200, filesize=16.5 K 2024-12-10T01:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/d69f6f27a4134ead8097a67462e78b17 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17 2024-12-10T01:16:37,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17, entries=150, sequenceid=200, filesize=11.9 K 2024-12-10T01:16:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/36762bf1f754467d9a4f32d60fb1cb36 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36 2024-12-10T01:16:37,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36, entries=150, sequenceid=200, filesize=11.9 K 2024-12-10T01:16:37,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for f7be4f2ce41631b00d075356558ca2ac in 459ms, sequenceid=200, compaction requested=false 2024-12-10T01:16:37,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:37,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:37,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/30814f399be7459789084af14cfa7fb6 is 50, key is test_row_0/A:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:37,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742257_1433 (size=14541) 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793397332, max timestamp=1733793397348Num uniques 859; 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793397332, max timestamp=1733793397348Num uniques 859; 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793397332, max timestamp=1733793397348Num uniques 859; 2024-12-10T01:16:37,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793457396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793457398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793457509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793457509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/30814f399be7459789084af14cfa7fb6 2024-12-10T01:16:37,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/a5549bf378014ae9865a25e16b11fb10 is 50, key is test_row_0/B:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:37,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742258_1434 (size=12151) 2024-12-10T01:16:37,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793457720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793457720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:38,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793458029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:38,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793458029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:38,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/a5549bf378014ae9865a25e16b11fb10 2024-12-10T01:16:38,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d30eda7efeab4f89809ec10e69d3b990 is 50, key is test_row_0/C:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:38,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742259_1435 (size=12151) 2024-12-10T01:16:38,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d30eda7efeab4f89809ec10e69d3b990 2024-12-10T01:16:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793458540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793458549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:38,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/30814f399be7459789084af14cfa7fb6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6 2024-12-10T01:16:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6, entries=200, sequenceid=213, filesize=14.2 K 2024-12-10T01:16:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/a5549bf378014ae9865a25e16b11fb10 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10 2024-12-10T01:16:38,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10, entries=150, sequenceid=213, filesize=11.9 K 2024-12-10T01:16:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d30eda7efeab4f89809ec10e69d3b990 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990 2024-12-10T01:16:38,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990, entries=150, sequenceid=213, filesize=11.9 K 2024-12-10T01:16:38,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f7be4f2ce41631b00d075356558ca2ac in 1298ms, sequenceid=213, compaction requested=true 2024-12-10T01:16:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T01:16:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T01:16:38,592 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:38,592 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:38,592 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:38,592 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:38,592 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:38,593 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/56fe40d051a646fbaec83e10155edd66, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=43.0 K 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 56fe40d051a646fbaec83e10155edd66, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793393202 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cded33fdab74febacc593f515c77bef, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793395446 2024-12-10T01:16:38,593 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:38,593 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/3f580d36be164aee9b205dd09aafabaa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=36.0 K 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 30814f399be7459789084af14cfa7fb6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:38,593 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f580d36be164aee9b205dd09aafabaa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793393202 2024-12-10T01:16:38,594 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36762bf1f754467d9a4f32d60fb1cb36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793395446 2024-12-10T01:16:38,594 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30eda7efeab4f89809ec10e69d3b990, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:38,598 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:38,599 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/dcf8c54f1a0849dfbf3fc9cbe0964835 is 50, key is test_row_0/A:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:38,599 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:38,599 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d70c86343f5c4a1e83f8bcd708a4b64a is 50, key is test_row_0/C:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:38,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742260_1436 (size=12663) 2024-12-10T01:16:38,607 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/d70c86343f5c4a1e83f8bcd708a4b64a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d70c86343f5c4a1e83f8bcd708a4b64a 2024-12-10T01:16:38,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742261_1437 (size=12663) 2024-12-10T01:16:38,611 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into d70c86343f5c4a1e83f8bcd708a4b64a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:38,611 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:38,611 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793398592; duration=0sec 2024-12-10T01:16:38,612 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:38,612 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:38,612 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:38,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:38,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:38,614 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:38,614 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d851fbfd14f4475588dc0e1adfd38dcd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=36.0 K 2024-12-10T01:16:38,614 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d851fbfd14f4475588dc0e1adfd38dcd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733793393202 2024-12-10T01:16:38,615 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69f6f27a4134ead8097a67462e78b17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733793395446 2024-12-10T01:16:38,616 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5549bf378014ae9865a25e16b11fb10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:38,624 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#371 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:38,625 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/904f7270050a4cadbc5a1276b06d6636 is 50, key is test_row_0/B:col10/1733793397291/Put/seqid=0 2024-12-10T01:16:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742262_1438 (size=12663) 2024-12-10T01:16:38,643 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/904f7270050a4cadbc5a1276b06d6636 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/904f7270050a4cadbc5a1276b06d6636 2024-12-10T01:16:38,648 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into 904f7270050a4cadbc5a1276b06d6636(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:38,648 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:38,648 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793398591; duration=0sec 2024-12-10T01:16:38,648 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:38,648 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:39,014 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/dcf8c54f1a0849dfbf3fc9cbe0964835 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/dcf8c54f1a0849dfbf3fc9cbe0964835 2024-12-10T01:16:39,017 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into dcf8c54f1a0849dfbf3fc9cbe0964835(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:39,018 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:39,018 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793398591; duration=0sec 2024-12-10T01:16:39,018 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:39,018 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T01:16:39,080 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:39,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-10T01:16:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:39,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:39,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:39,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:39,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-10T01:16:39,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:39,234 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/54ee0d4ec74c429da1ff9e108bec6327 is 50, key is test_row_0/A:col10/1733793397395/Put/seqid=0 2024-12-10T01:16:39,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742263_1439 (size=12151) 2024-12-10T01:16:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:39,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793459578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:39,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793459580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:39,640 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/54ee0d4ec74c429da1ff9e108bec6327 2024-12-10T01:16:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f975178d2b0342c5b4b27fd3beb5e8ec is 50, key is test_row_0/B:col10/1733793397395/Put/seqid=0 2024-12-10T01:16:39,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742264_1440 (size=12151) 2024-12-10T01:16:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793459689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793459689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793459899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793459899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,050 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f975178d2b0342c5b4b27fd3beb5e8ec 2024-12-10T01:16:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/0d01b280d3164d07ad6b896336c78ef8 is 50, key is test_row_0/C:col10/1733793397395/Put/seqid=0 2024-12-10T01:16:40,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742265_1441 (size=12151) 2024-12-10T01:16:40,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793460209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:40,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793460220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48940 deadline: 1733793460429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,433 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48874 deadline: 1733793460439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48888 deadline: 1733793460439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,446 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:40,446 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in f7be4f2ce41631b00d075356558ca2ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,472 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/0d01b280d3164d07ad6b896336c78ef8 2024-12-10T01:16:40,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/54ee0d4ec74c429da1ff9e108bec6327 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327 2024-12-10T01:16:40,478 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T01:16:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/f975178d2b0342c5b4b27fd3beb5e8ec as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec 2024-12-10T01:16:40,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T01:16:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/0d01b280d3164d07ad6b896336c78ef8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8 2024-12-10T01:16:40,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T01:16:40,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f7be4f2ce41631b00d075356558ca2ac in 1252ms, sequenceid=253, compaction requested=false 2024-12-10T01:16:40,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:40,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:40,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-10T01:16:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-10T01:16:40,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-10T01:16:40,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-12-10T01:16:40,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4090 sec 2024-12-10T01:16:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:40,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:40,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4e66262c04f64de0886ec824bd510a74 is 50, key is test_row_0/A:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742266_1442 (size=12251) 2024-12-10T01:16:40,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4e66262c04f64de0886ec824bd510a74 2024-12-10T01:16:40,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/86a9d6af4d9641e0a6b9ffdccd757c14 is 50, key is test_row_0/B:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:40,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742267_1443 (size=12251) 2024-12-10T01:16:40,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/86a9d6af4d9641e0a6b9ffdccd757c14 2024-12-10T01:16:40,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/dbed2fd6157a4bd3a421fea4341d659e is 50, key is test_row_0/C:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:40,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742268_1444 (size=12251) 2024-12-10T01:16:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793400773, max timestamp=1733793400811Num uniques 859; 2024-12-10T01:16:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793400773, max timestamp=1733793400811Num uniques 859; 2024-12-10T01:16:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:40,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733793400773, max timestamp=1733793400811Num uniques 859; 2024-12-10T01:16:40,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T01:16:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793460870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793460871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793460980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:40,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793460989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/dbed2fd6157a4bd3a421fea4341d659e 2024-12-10T01:16:41,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4e66262c04f64de0886ec824bd510a74 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74 2024-12-10T01:16:41,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T01:16:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/86a9d6af4d9641e0a6b9ffdccd757c14 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14 2024-12-10T01:16:41,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T01:16:41,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/dbed2fd6157a4bd3a421fea4341d659e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e 2024-12-10T01:16:41,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T01:16:41,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f7be4f2ce41631b00d075356558ca2ac in 452ms, sequenceid=264, compaction requested=true 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:41,175 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:41,175 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:41,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:41,176 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,176 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,176 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/904f7270050a4cadbc5a1276b06d6636, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=36.2 K 2024-12-10T01:16:41,176 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/dcf8c54f1a0849dfbf3fc9cbe0964835, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=36.2 K 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 904f7270050a4cadbc5a1276b06d6636, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:41,176 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf8c54f1a0849dfbf3fc9cbe0964835, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:41,177 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ee0d4ec74c429da1ff9e108bec6327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733793397390 2024-12-10T01:16:41,177 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f975178d2b0342c5b4b27fd3beb5e8ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733793397390 2024-12-10T01:16:41,177 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e66262c04f64de0886ec824bd510a74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:41,177 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a9d6af4d9641e0a6b9ffdccd757c14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:41,183 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:41,184 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/144efa3fbd5c4dc9aa8172fb59745ee2 is 50, key is test_row_0/A:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:41,190 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:41,191 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/180dfdbe56fb4b95abef8804d40f265e is 50, key is test_row_0/B:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742269_1445 (size=12865) 2024-12-10T01:16:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:41,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:41,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/429b6748217c487c9c6e1dcba6593e02 is 50, key is test_row_0/A:col10/1733793400866/Put/seqid=0 2024-12-10T01:16:41,202 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/144efa3fbd5c4dc9aa8172fb59745ee2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/144efa3fbd5c4dc9aa8172fb59745ee2 2024-12-10T01:16:41,208 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into 144efa3fbd5c4dc9aa8172fb59745ee2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:41,208 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:41,208 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=13, startTime=1733793401175; duration=0sec 2024-12-10T01:16:41,209 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:41,209 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:41,209 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:16:41,211 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:16:41,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:41,212 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,212 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d70c86343f5c4a1e83f8bcd708a4b64a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=36.2 K 2024-12-10T01:16:41,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d70c86343f5c4a1e83f8bcd708a4b64a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733793396631 2024-12-10T01:16:41,212 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d01b280d3164d07ad6b896336c78ef8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733793397390 2024-12-10T01:16:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742271_1447 (size=14741) 2024-12-10T01:16:41,213 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbed2fd6157a4bd3a421fea4341d659e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:41,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/429b6748217c487c9c6e1dcba6593e02 2024-12-10T01:16:41,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742270_1446 (size=12865) 2024-12-10T01:16:41,222 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:41,223 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a is 50, key is test_row_0/C:col10/1733793399570/Put/seqid=0 2024-12-10T01:16:41,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5c1e4408150445c290195c0cdce4bbbd is 50, key is test_row_0/B:col10/1733793400866/Put/seqid=0 2024-12-10T01:16:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T01:16:41,229 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:41,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:41,231 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/180dfdbe56fb4b95abef8804d40f265e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/180dfdbe56fb4b95abef8804d40f265e 2024-12-10T01:16:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-10T01:16:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:41,232 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:41,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:41,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:41,237 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into 180dfdbe56fb4b95abef8804d40f265e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:41,237 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:41,237 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=13, startTime=1733793401175; duration=0sec 2024-12-10T01:16:41,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:41,238 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793461228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793461230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742272_1448 (size=12865) 2024-12-10T01:16:41,256 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a 2024-12-10T01:16:41,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742273_1449 (size=12301) 2024-12-10T01:16:41,261 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into 5d38cdb0b2f5431fb1e2d5fbb5e3ff4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:41,261 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:41,262 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=13, startTime=1733793401175; duration=0sec 2024-12-10T01:16:41,262 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:41,262 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:41,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793461349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793461349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:41,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:41,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793461559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793461559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5c1e4408150445c290195c0cdce4bbbd 2024-12-10T01:16:41,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/1b9317b179fc4cbcbf404edc7001d428 is 50, key is test_row_0/C:col10/1733793400866/Put/seqid=0 2024-12-10T01:16:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742274_1450 (size=12301) 2024-12-10T01:16:41,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:41,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:41,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:41,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793461869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793461869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:41,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:41,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:42,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/1b9317b179fc4cbcbf404edc7001d428 2024-12-10T01:16:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/429b6748217c487c9c6e1dcba6593e02 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02 2024-12-10T01:16:42,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02, entries=200, sequenceid=301, filesize=14.4 K 2024-12-10T01:16:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5c1e4408150445c290195c0cdce4bbbd as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd 2024-12-10T01:16:42,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd, entries=150, sequenceid=301, filesize=12.0 K 2024-12-10T01:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/1b9317b179fc4cbcbf404edc7001d428 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428 2024-12-10T01:16:42,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428, entries=150, sequenceid=301, filesize=12.0 K 2024-12-10T01:16:42,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f7be4f2ce41631b00d075356558ca2ac in 895ms, sequenceid=301, compaction requested=false 2024-12-10T01:16:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:42,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T01:16:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:42,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4eb5b6cf6c6d4d5da33ed020e47f6bec is 50, key is test_row_0/A:col10/1733793401228/Put/seqid=0 2024-12-10T01:16:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742275_1451 (size=12301) 2024-12-10T01:16:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:42,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793402412, max timestamp=1733793402422Num uniques 859; 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793402412, max timestamp=1733793402422Num uniques 859; 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793402412, max timestamp=1733793402422Num uniques 859; 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:42,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:16:42,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793462465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793462468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,555 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4eb5b6cf6c6d4d5da33ed020e47f6bec 2024-12-10T01:16:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/6fdd755a95264fafadfddf8829037d3a is 50, key is test_row_0/B:col10/1733793401228/Put/seqid=0 2024-12-10T01:16:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742276_1452 (size=12301) 2024-12-10T01:16:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793462579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793462579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793462789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793462790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:42,964 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/6fdd755a95264fafadfddf8829037d3a 2024-12-10T01:16:42,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/cc6b7052159744719ea07f65ac23ce09 is 50, key is test_row_0/C:col10/1733793401228/Put/seqid=0 2024-12-10T01:16:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742277_1453 (size=12301) 2024-12-10T01:16:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793463099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793463099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/cc6b7052159744719ea07f65ac23ce09 2024-12-10T01:16:43,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4eb5b6cf6c6d4d5da33ed020e47f6bec as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec 2024-12-10T01:16:43,379 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec, entries=150, sequenceid=314, filesize=12.0 K 2024-12-10T01:16:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/6fdd755a95264fafadfddf8829037d3a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a 2024-12-10T01:16:43,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a, entries=150, sequenceid=314, filesize=12.0 K 2024-12-10T01:16:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/cc6b7052159744719ea07f65ac23ce09 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09 2024-12-10T01:16:43,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09, entries=150, sequenceid=314, filesize=12.0 K 2024-12-10T01:16:43,387 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f7be4f2ce41631b00d075356558ca2ac in 1241ms, sequenceid=314, compaction requested=true 2024-12-10T01:16:43,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:43,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:43,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-10T01:16:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-10T01:16:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:43,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-10T01:16:43,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1550 sec 2024-12-10T01:16:43,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.1620 sec 2024-12-10T01:16:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:43,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:16:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793463633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793463638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4230425ba14045b48c11e8729664ea71 is 50, key is test_row_0/A:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:43,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742278_1454 (size=14741) 2024-12-10T01:16:43,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793463749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793463749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793463959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793463959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:44,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4230425ba14045b48c11e8729664ea71 2024-12-10T01:16:44,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/0b84624046804dc3975efce6399af699 is 50, key is test_row_0/B:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:44,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742279_1455 (size=12301) 2024-12-10T01:16:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793464269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:44,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793464279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:44,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/0b84624046804dc3975efce6399af699 2024-12-10T01:16:44,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/534d6feb413b443090c425bdfbe94dd8 is 50, key is test_row_0/C:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:44,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742280_1456 (size=12301) 2024-12-10T01:16:44,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48922 deadline: 1733793464779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:44,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48948 deadline: 1733793464789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 , the old value is region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=2 because the exception is null or not the one we care about 2024-12-10T01:16:44,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/534d6feb413b443090c425bdfbe94dd8 2024-12-10T01:16:44,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/4230425ba14045b48c11e8729664ea71 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71 2024-12-10T01:16:44,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71, entries=200, sequenceid=351, filesize=14.4 K 2024-12-10T01:16:44,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/0b84624046804dc3975efce6399af699 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699 2024-12-10T01:16:44,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T01:16:44,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/534d6feb413b443090c425bdfbe94dd8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8 2024-12-10T01:16:44,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T01:16:44,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f7be4f2ce41631b00d075356558ca2ac in 1294ms, sequenceid=351, compaction requested=true 2024-12-10T01:16:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:16:44,908 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:16:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:44,909 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7be4f2ce41631b00d075356558ca2ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:16:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:44,909 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:44,909 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:44,909 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/B is initiating minor compaction (all files) 2024-12-10T01:16:44,909 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/A is initiating minor compaction (all files) 2024-12-10T01:16:44,909 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/B in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:44,909 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/A in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:44,909 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/180dfdbe56fb4b95abef8804d40f265e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=48.6 K 2024-12-10T01:16:44,909 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/144efa3fbd5c4dc9aa8172fb59745ee2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=53.4 K 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 180dfdbe56fb4b95abef8804d40f265e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 144efa3fbd5c4dc9aa8172fb59745ee2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429b6748217c487c9c6e1dcba6593e02, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733793400844 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1e4408150445c290195c0cdce4bbbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733793400844 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb5b6cf6c6d4d5da33ed020e47f6bec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733793401205 2024-12-10T01:16:44,910 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fdd755a95264fafadfddf8829037d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733793401205 2024-12-10T01:16:44,911 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4230425ba14045b48c11e8729664ea71, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733793402455 2024-12-10T01:16:44,911 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b84624046804dc3975efce6399af699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733793402455 2024-12-10T01:16:44,918 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:44,918 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/c0657808890e421f8552dd8a5091e768 is 50, key is test_row_0/B:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:44,919 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:44,919 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/5f08d160b22244309f9a1666fde97ca7 is 50, key is test_row_0/A:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:44,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742281_1457 (size=13051) 2024-12-10T01:16:44,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742282_1458 (size=13051) 2024-12-10T01:16:44,924 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/c0657808890e421f8552dd8a5091e768 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/c0657808890e421f8552dd8a5091e768 2024-12-10T01:16:44,927 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/B of f7be4f2ce41631b00d075356558ca2ac into c0657808890e421f8552dd8a5091e768(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:44,927 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:44,927 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/B, priority=12, startTime=1733793404908; duration=0sec 2024-12-10T01:16:44,927 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:16:44,927 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:B 2024-12-10T01:16:44,927 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T01:16:44,928 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T01:16:44,928 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): f7be4f2ce41631b00d075356558ca2ac/C is initiating minor compaction (all files) 2024-12-10T01:16:44,928 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7be4f2ce41631b00d075356558ca2ac/C in TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:44,928 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp, totalSize=48.6 K 2024-12-10T01:16:44,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d38cdb0b2f5431fb1e2d5fbb5e3ff4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733793399561 2024-12-10T01:16:44,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9317b179fc4cbcbf404edc7001d428, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733793400844 2024-12-10T01:16:44,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6b7052159744719ea07f65ac23ce09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733793401205 2024-12-10T01:16:44,929 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 534d6feb413b443090c425bdfbe94dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733793402455 2024-12-10T01:16:44,935 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7be4f2ce41631b00d075356558ca2ac#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:16:44,935 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/b70e1cebb0144f15873622d93a7cdfd5 is 50, key is test_row_0/C:col10/1733793403613/Put/seqid=0 2024-12-10T01:16:44,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742283_1459 (size=13051) 2024-12-10T01:16:45,327 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/5f08d160b22244309f9a1666fde97ca7 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/5f08d160b22244309f9a1666fde97ca7 2024-12-10T01:16:45,330 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/A of f7be4f2ce41631b00d075356558ca2ac into 5f08d160b22244309f9a1666fde97ca7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:45,330 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:45,330 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/A, priority=12, startTime=1733793404908; duration=0sec 2024-12-10T01:16:45,330 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:45,330 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:A 2024-12-10T01:16:45,341 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/b70e1cebb0144f15873622d93a7cdfd5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/b70e1cebb0144f15873622d93a7cdfd5 2024-12-10T01:16:45,345 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7be4f2ce41631b00d075356558ca2ac/C of f7be4f2ce41631b00d075356558ca2ac into b70e1cebb0144f15873622d93a7cdfd5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:16:45,345 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:45,345 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., storeName=f7be4f2ce41631b00d075356558ca2ac/C, priority=12, startTime=1733793404909; duration=0sec 2024-12-10T01:16:45,345 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:16:45,345 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7be4f2ce41631b00d075356558ca2ac:C 2024-12-10T01:16:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T01:16:45,400 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:45,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-10T01:16:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:45,402 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:45,403 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:45,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:45,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:45,556 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:45,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/264dd213a84542bab2c14f40aa4aa413 is 50, key is test_row_0/A:col10/1733793403626/Put/seqid=0 2024-12-10T01:16:45,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742284_1460 (size=12301) 2024-12-10T01:16:45,572 INFO [Thread-1793 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1793. 2024-12-10T01:16:45,572 DEBUG [Thread-1793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,572 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,572 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,574 INFO [Thread-1789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1789. 2024-12-10T01:16:45,574 DEBUG [Thread-1789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,574 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,574 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,574 INFO [Thread-1787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1787. 2024-12-10T01:16:45,574 DEBUG [Thread-1787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,574 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,574 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,577 INFO [Thread-1785 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1785. 2024-12-10T01:16:45,577 DEBUG [Thread-1785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,577 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,577 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,578 INFO [Thread-1791 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1791. 2024-12-10T01:16:45,578 DEBUG [Thread-1791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,578 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,578 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. as already flushing 2024-12-10T01:16:45,794 INFO [Thread-1774 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1774. 2024-12-10T01:16:45,794 DEBUG [Thread-1774 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,794 DEBUG [Thread-1774 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,794 DEBUG [Thread-1774 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,812 INFO [Thread-1776 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1776. 2024-12-10T01:16:45,812 DEBUG [Thread-1776 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:45,812 DEBUG [Thread-1776 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,812 DEBUG [Thread-1776 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:45,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:45,965 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/264dd213a84542bab2c14f40aa4aa413 2024-12-10T01:16:45,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5dd8144a6c994a19b59455d28c828cfa is 50, key is test_row_0/B:col10/1733793403626/Put/seqid=0 2024-12-10T01:16:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742285_1461 (size=12301) 2024-12-10T01:16:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:46,380 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5dd8144a6c994a19b59455d28c828cfa 2024-12-10T01:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/ad12bb8807664f70a49deed8eb5ddbd1 is 50, key is test_row_0/C:col10/1733793403626/Put/seqid=0 2024-12-10T01:16:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742286_1462 (size=12301) 2024-12-10T01:16:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:46,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/ad12bb8807664f70a49deed8eb5ddbd1 2024-12-10T01:16:46,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/264dd213a84542bab2c14f40aa4aa413 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/264dd213a84542bab2c14f40aa4aa413 2024-12-10T01:16:46,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/264dd213a84542bab2c14f40aa4aa413, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T01:16:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/5dd8144a6c994a19b59455d28c828cfa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5dd8144a6c994a19b59455d28c828cfa 2024-12-10T01:16:46,816 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5dd8144a6c994a19b59455d28c828cfa, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T01:16:46,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/ad12bb8807664f70a49deed8eb5ddbd1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/ad12bb8807664f70a49deed8eb5ddbd1 2024-12-10T01:16:46,820 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/ad12bb8807664f70a49deed8eb5ddbd1, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T01:16:46,821 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=13.42 KB/13740 for f7be4f2ce41631b00d075356558ca2ac in 1265ms, sequenceid=363, compaction requested=false 2024-12-10T01:16:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for f7be4f2ce41631b00d075356558ca2ac: 2024-12-10T01:16:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-10T01:16:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-10T01:16:46,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-10T01:16:46,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-12-10T01:16:46,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4230 sec 2024-12-10T01:16:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T01:16:47,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:50,474 INFO [Thread-1782 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1782. 2024-12-10T01:16:50,474 DEBUG [Thread-1782 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:50,474 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,475 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:50,533 INFO [Thread-1778 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1778. 2024-12-10T01:16:50,534 DEBUG [Thread-1778 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:50,534 DEBUG [Thread-1778 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,534 DEBUG [Thread-1778 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:50,542 INFO [Thread-1780 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1780. 2024-12-10T01:16:50,542 DEBUG [Thread-1780 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:16:50,542 DEBUG [Thread-1780 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,542 DEBUG [Thread-1780 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1760 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5275 rows 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1767 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5299 rows 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1764 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5290 rows 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1766 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5293 rows 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1762 2024-12-10T01:16:50,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5284 rows 2024-12-10T01:16:50,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:16:50,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:16:50,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:50,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:16:50,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:16:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:50,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793410551"}]},"ts":"1733793410551"} 2024-12-10T01:16:50,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:16:50,553 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:16:50,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:16:50,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, UNASSIGN}] 2024-12-10T01:16:50,556 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, UNASSIGN 2024-12-10T01:16:50,557 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f7be4f2ce41631b00d075356558ca2ac, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:50,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, UNASSIGN because future has completed 2024-12-10T01:16:50,558 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:16:50,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:50,710 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:50,710 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing f7be4f2ce41631b00d075356558ca2ac, disabling compactions & flushes 2024-12-10T01:16:50,711 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. after waiting 0 ms 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:50,711 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing f7be4f2ce41631b00d075356558ca2ac 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=A 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=B 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7be4f2ce41631b00d075356558ca2ac, store=C 2024-12-10T01:16:50,711 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:50,714 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/609b6b0c3c664e1090e5f067ea1ce371 is 50, key is test_row_0/A:col10/1733793410540/Put/seqid=0 2024-12-10T01:16:50,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742287_1463 (size=9857) 2024-12-10T01:16:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:51,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:16:51,118 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/609b6b0c3c664e1090e5f067ea1ce371 2024-12-10T01:16:51,128 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/edd7b05faa134acd9123312a288939cf is 50, key is test_row_0/B:col10/1733793410540/Put/seqid=0 2024-12-10T01:16:51,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742288_1464 (size=9857) 2024-12-10T01:16:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:51,534 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/edd7b05faa134acd9123312a288939cf 2024-12-10T01:16:51,548 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/f48d53119a08444aba83c4a5c587be8d is 50, key is test_row_0/C:col10/1733793410540/Put/seqid=0 2024-12-10T01:16:51,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742289_1465 (size=9857) 2024-12-10T01:16:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:51,953 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/f48d53119a08444aba83c4a5c587be8d 2024-12-10T01:16:51,963 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/A/609b6b0c3c664e1090e5f067ea1ce371 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/609b6b0c3c664e1090e5f067ea1ce371 2024-12-10T01:16:51,966 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/609b6b0c3c664e1090e5f067ea1ce371, entries=100, sequenceid=371, filesize=9.6 K 2024-12-10T01:16:51,967 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/B/edd7b05faa134acd9123312a288939cf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/edd7b05faa134acd9123312a288939cf 2024-12-10T01:16:51,969 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/edd7b05faa134acd9123312a288939cf, entries=100, sequenceid=371, filesize=9.6 K 2024-12-10T01:16:51,970 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/.tmp/C/f48d53119a08444aba83c4a5c587be8d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/f48d53119a08444aba83c4a5c587be8d 2024-12-10T01:16:51,972 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/f48d53119a08444aba83c4a5c587be8d, entries=100, sequenceid=371, filesize=9.6 K 2024-12-10T01:16:51,973 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f7be4f2ce41631b00d075356558ca2ac in 1261ms, sequenceid=371, compaction requested=true 2024-12-10T01:16:51,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/8c5ab5919674497db2a9a2674cb82f51, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a1f91d13303b4a70a39b4b72aaf89f6f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/e56113fb0a654f15bca03e8bd6c8d2a4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/56fe40d051a646fbaec83e10155edd66, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/dcf8c54f1a0849dfbf3fc9cbe0964835, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/144efa3fbd5c4dc9aa8172fb59745ee2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71] to archive 2024-12-10T01:16:51,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/8c5ab5919674497db2a9a2674cb82f51 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/8c5ab5919674497db2a9a2674cb82f51 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/306fa0b45dcc4cc499c79f07c5f82e03 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/80f135da44fc449c987a7f7b57987773 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a4c757cd9e504f51a0d33e1d5347246d 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/2c31ffc1fa7a4feb8034b55ad8d394e1 2024-12-10T01:16:51,975 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/29cfbeb8461c46b583a2291f570c3461 2024-12-10T01:16:51,976 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a1f91d13303b4a70a39b4b72aaf89f6f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/a1f91d13303b4a70a39b4b72aaf89f6f 2024-12-10T01:16:51,976 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/f84732cd60054025acc237488fb47aa0 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/763ab15837504bf0abc31bb73f3c768b 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/d42c6ab292b14a48969e0093a81083a1 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/e56113fb0a654f15bca03e8bd6c8d2a4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/e56113fb0a654f15bca03e8bd6c8d2a4 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4cded33fdab74febacc593f515c77bef 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/522ebfee3539445e8791bdd09c2d5b2a 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/56fe40d051a646fbaec83e10155edd66 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/56fe40d051a646fbaec83e10155edd66 2024-12-10T01:16:51,977 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/30814f399be7459789084af14cfa7fb6 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/dcf8c54f1a0849dfbf3fc9cbe0964835 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/dcf8c54f1a0849dfbf3fc9cbe0964835 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/54ee0d4ec74c429da1ff9e108bec6327 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/144efa3fbd5c4dc9aa8172fb59745ee2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/144efa3fbd5c4dc9aa8172fb59745ee2 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4e66262c04f64de0886ec824bd510a74 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/429b6748217c487c9c6e1dcba6593e02 2024-12-10T01:16:51,978 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4eb5b6cf6c6d4d5da33ed020e47f6bec 2024-12-10T01:16:51,979 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/4230425ba14045b48c11e8729664ea71 2024-12-10T01:16:51,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/bfe28543486b4d49a5cee2402dbec7b5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/25ac9922a3194070829e7dc4e92a4955, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/8a129b5388a34b728e7c721003a9df2b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d851fbfd14f4475588dc0e1adfd38dcd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/904f7270050a4cadbc5a1276b06d6636, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/180dfdbe56fb4b95abef8804d40f265e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699] to archive 2024-12-10T01:16:51,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d9672e94448e42cc8cb17474f8a796dd 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/418df6de2960410fadbb53324f0453ac 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/19dc0f574c514c55bb2f9849a562fcb6 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f39bd47df30342b8a4bb7dd733d07ba3 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7c0e30f0388747f8a1842c6082e50944 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/25ac9922a3194070829e7dc4e92a4955 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/25ac9922a3194070829e7dc4e92a4955 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/7b1989ea096742189d82858217cba2f1 2024-12-10T01:16:51,982 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/bfe28543486b4d49a5cee2402dbec7b5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/bfe28543486b4d49a5cee2402dbec7b5 2024-12-10T01:16:51,983 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/8a129b5388a34b728e7c721003a9df2b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/8a129b5388a34b728e7c721003a9df2b 2024-12-10T01:16:51,983 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/11679408809a41269a5eee771036f7b1 2024-12-10T01:16:51,983 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/61b71dc3e6414cd19133f6db1c61e764 2024-12-10T01:16:51,983 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/9d4986c0fdbd465e8393b468dd801617 2024-12-10T01:16:51,983 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d69f6f27a4134ead8097a67462e78b17 2024-12-10T01:16:51,984 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d851fbfd14f4475588dc0e1adfd38dcd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/d851fbfd14f4475588dc0e1adfd38dcd 2024-12-10T01:16:51,984 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/904f7270050a4cadbc5a1276b06d6636 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/904f7270050a4cadbc5a1276b06d6636 2024-12-10T01:16:51,984 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/a5549bf378014ae9865a25e16b11fb10 2024-12-10T01:16:51,984 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/180dfdbe56fb4b95abef8804d40f265e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/180dfdbe56fb4b95abef8804d40f265e 2024-12-10T01:16:51,984 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5c1e4408150445c290195c0cdce4bbbd 2024-12-10T01:16:51,985 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/86a9d6af4d9641e0a6b9ffdccd757c14 2024-12-10T01:16:51,985 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/f975178d2b0342c5b4b27fd3beb5e8ec 2024-12-10T01:16:51,985 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/0b84624046804dc3975efce6399af699 2024-12-10T01:16:51,985 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/6fdd755a95264fafadfddf8829037d3a 2024-12-10T01:16:51,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e3743006362346d0bbab48b5b3af6400, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/74f43a9dd7d840c8bb4a746d355fca80, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/35c8f6273d6246c3bce678048a931a8c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/3f580d36be164aee9b205dd09aafabaa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d70c86343f5c4a1e83f8bcd708a4b64a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8] to archive 2024-12-10T01:16:51,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:16:51,987 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/350aafc3a49f40b5ae7c289874c5ecf0 2024-12-10T01:16:51,987 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/eca253fedaa841fdbfea1af742a843c4 2024-12-10T01:16:51,987 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/26c7bf362fef4b37be08b138fc7f0934 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e3743006362346d0bbab48b5b3af6400 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e3743006362346d0bbab48b5b3af6400 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/74f43a9dd7d840c8bb4a746d355fca80 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/74f43a9dd7d840c8bb4a746d355fca80 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d68caaf8088d4ec88e7f492ed8c2e799 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/57c416c7400b4822be7d805e836c3229 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/8e874e1cc2c34395a0ad82c873825a68 2024-12-10T01:16:51,988 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/35c8f6273d6246c3bce678048a931a8c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/35c8f6273d6246c3bce678048a931a8c 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/e0702c695db34a26bc1b8464dc0efe9e 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/9bdb7bae50f2461898cae5bdc53760cf 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/3f580d36be164aee9b205dd09aafabaa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/3f580d36be164aee9b205dd09aafabaa 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d70c86343f5c4a1e83f8bcd708a4b64a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d70c86343f5c4a1e83f8bcd708a4b64a 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/d30eda7efeab4f89809ec10e69d3b990 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/36762bf1f754467d9a4f32d60fb1cb36 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/67cd7580278e46b0b9dd437443dbe521 2024-12-10T01:16:51,989 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/0d01b280d3164d07ad6b896336c78ef8 2024-12-10T01:16:51,990 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/5d38cdb0b2f5431fb1e2d5fbb5e3ff4a 2024-12-10T01:16:51,990 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/dbed2fd6157a4bd3a421fea4341d659e 2024-12-10T01:16:51,990 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/cc6b7052159744719ea07f65ac23ce09 2024-12-10T01:16:51,990 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/534d6feb413b443090c425bdfbe94dd8 2024-12-10T01:16:51,990 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/1b9317b179fc4cbcbf404edc7001d428 2024-12-10T01:16:51,993 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/recovered.edits/374.seqid, newMaxSeqId=374, maxSeqId=1 2024-12-10T01:16:51,993 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. 2024-12-10T01:16:51,993 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for f7be4f2ce41631b00d075356558ca2ac: Waiting for close lock at 1733793410711Running coprocessor pre-close hooks at 1733793410711Disabling compacts and flushes for region at 1733793410711Disabling writes for close at 1733793410711Obtaining lock to block concurrent updates at 1733793410711Preparing flush snapshotting stores in f7be4f2ce41631b00d075356558ca2ac at 1733793410711Finished memstore snapshotting TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733793410711Flushing stores of TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac. at 1733793410711Flushing f7be4f2ce41631b00d075356558ca2ac/A: creating writer at 1733793410711Flushing f7be4f2ce41631b00d075356558ca2ac/A: appending metadata at 1733793410714 (+3 ms)Flushing f7be4f2ce41631b00d075356558ca2ac/A: closing flushed file at 1733793410714Flushing f7be4f2ce41631b00d075356558ca2ac/B: creating writer at 1733793411124 (+410 ms)Flushing f7be4f2ce41631b00d075356558ca2ac/B: appending metadata at 1733793411127 (+3 ms)Flushing f7be4f2ce41631b00d075356558ca2ac/B: closing flushed file at 1733793411127Flushing f7be4f2ce41631b00d075356558ca2ac/C: creating writer at 1733793411543 (+416 ms)Flushing f7be4f2ce41631b00d075356558ca2ac/C: appending metadata at 1733793411547 (+4 ms)Flushing f7be4f2ce41631b00d075356558ca2ac/C: closing flushed file at 1733793411547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f6649b6: reopening flushed file at 1733793411962 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43b215ec: reopening flushed file at 1733793411966 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@138d1fe: reopening flushed file at 1733793411969 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f7be4f2ce41631b00d075356558ca2ac in 1261ms, sequenceid=371, compaction requested=true at 1733793411973 (+4 ms)Writing region close event to WAL at 1733793411991 (+18 ms)Running coprocessor post-close hooks at 1733793411993 (+2 ms)Closed at 1733793411993 2024-12-10T01:16:51,995 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:51,995 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f7be4f2ce41631b00d075356558ca2ac, regionState=CLOSED 2024-12-10T01:16:51,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:51,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-10T01:16:51,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure f7be4f2ce41631b00d075356558ca2ac, server=fd1a2b66caf8,38925,1733793262612 in 1.4390 sec 2024-12-10T01:16:51,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-10T01:16:51,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7be4f2ce41631b00d075356558ca2ac, UNASSIGN in 1.4430 sec 2024-12-10T01:16:52,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-10T01:16:52,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4460 sec 2024-12-10T01:16:52,002 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793412002"}]},"ts":"1733793412002"} 2024-12-10T01:16:52,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:16:52,003 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:16:52,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4590 sec 2024-12-10T01:16:52,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T01:16:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T01:16:52,711 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:52,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:16:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,717 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T01:16:52,718 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,721 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:52,725 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/recovered.edits] 2024-12-10T01:16:52,731 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/264dd213a84542bab2c14f40aa4aa413 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/264dd213a84542bab2c14f40aa4aa413 2024-12-10T01:16:52,731 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/609b6b0c3c664e1090e5f067ea1ce371 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/609b6b0c3c664e1090e5f067ea1ce371 2024-12-10T01:16:52,731 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/5f08d160b22244309f9a1666fde97ca7 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/A/5f08d160b22244309f9a1666fde97ca7 2024-12-10T01:16:52,734 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/c0657808890e421f8552dd8a5091e768 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/c0657808890e421f8552dd8a5091e768 2024-12-10T01:16:52,734 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5dd8144a6c994a19b59455d28c828cfa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/5dd8144a6c994a19b59455d28c828cfa 2024-12-10T01:16:52,735 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/edd7b05faa134acd9123312a288939cf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/B/edd7b05faa134acd9123312a288939cf 2024-12-10T01:16:52,737 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/ad12bb8807664f70a49deed8eb5ddbd1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/ad12bb8807664f70a49deed8eb5ddbd1 2024-12-10T01:16:52,737 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/f48d53119a08444aba83c4a5c587be8d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/f48d53119a08444aba83c4a5c587be8d 2024-12-10T01:16:52,738 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/b70e1cebb0144f15873622d93a7cdfd5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/C/b70e1cebb0144f15873622d93a7cdfd5 2024-12-10T01:16:52,740 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/recovered.edits/374.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac/recovered.edits/374.seqid 2024-12-10T01:16:52,741 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/f7be4f2ce41631b00d075356558ca2ac 2024-12-10T01:16:52,741 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:16:52,744 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,746 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:16:52,749 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:16:52,750 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,750 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:16:52,750 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793412750"}]},"ts":"9223372036854775807"} 2024-12-10T01:16:52,752 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:16:52,752 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7be4f2ce41631b00d075356558ca2ac, NAME => 'TestAcidGuarantees,,1733793383364.f7be4f2ce41631b00d075356558ca2ac.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:16:52,753 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:16:52,753 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793412753"}]},"ts":"9223372036854775807"} 2024-12-10T01:16:52,755 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:16:52,756 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-10T01:16:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T01:16:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:16:52,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:52,842 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255 (was 257), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=7352 (was 7381) 2024-12-10T01:16:52,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=7351 2024-12-10T01:16:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:16:52,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:16:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:52,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T01:16:52,854 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:52,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-12-10T01:16:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:52,855 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T01:16:52,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742290_1466 (size=1061) 2024-12-10T01:16:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:53,265 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126 2024-12-10T01:16:53,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742291_1467 (size=53) 2024-12-10T01:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:53,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:16:53,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bff4d84e434d4670f70bf8a0a3136faf, disabling compactions & flushes 2024-12-10T01:16:53,674 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:53,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:53,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. after waiting 0 ms 2024-12-10T01:16:53,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:53,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:53,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bff4d84e434d4670f70bf8a0a3136faf: Waiting for close lock at 1733793413674Disabling compacts and flushes for region at 1733793413674Disabling writes for close at 1733793413674Writing region close event to WAL at 1733793413675 (+1 ms)Closed at 1733793413675 2024-12-10T01:16:53,677 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T01:16:53,677 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733793413677"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733793413677"}]},"ts":"1733793413677"} 2024-12-10T01:16:53,680 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T01:16:53,682 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T01:16:53,682 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793413682"}]},"ts":"1733793413682"} 2024-12-10T01:16:53,685 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T01:16:53,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, ASSIGN}] 2024-12-10T01:16:53,687 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, ASSIGN 2024-12-10T01:16:53,687 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, ASSIGN; state=OFFLINE, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=false 2024-12-10T01:16:53,838 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:53,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, ASSIGN because future has completed 2024-12-10T01:16:53,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:53,997 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:53,998 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:16:53,998 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:53,998 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:16:53,998 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:53,998 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,000 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,001 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:54,001 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName A 2024-12-10T01:16:54,001 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:54,002 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:54,002 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,003 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:54,003 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName B 2024-12-10T01:16:54,003 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:54,004 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:54,004 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,005 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:54,005 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName C 2024-12-10T01:16:54,005 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:54,006 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:54,006 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:54,006 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,007 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,007 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,009 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,009 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,009 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:16:54,011 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,013 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T01:16:54,013 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened bff4d84e434d4670f70bf8a0a3136faf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66912410, jitterRate=-0.0029273927211761475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:16:54,013 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:54,014 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for bff4d84e434d4670f70bf8a0a3136faf: Running coprocessor pre-open hook at 1733793413998Writing region info on filesystem at 1733793413998Initializing all the Stores at 1733793413999 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793413999Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793413999Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793413999Cleaning up temporary data from old regions at 1733793414009 (+10 ms)Running coprocessor post-open hooks at 1733793414013 (+4 ms)Region opened successfully at 1733793414014 (+1 ms) 2024-12-10T01:16:54,015 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., pid=124, masterSystemTime=1733793413991 2024-12-10T01:16:54,017 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:54,017 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:54,018 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=OPEN, openSeqNum=2, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:54,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:54,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-10T01:16:54,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 in 182 msec 2024-12-10T01:16:54,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-12-10T01:16:54,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, ASSIGN in 338 msec 2024-12-10T01:16:54,028 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T01:16:54,028 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793414028"}]},"ts":"1733793414028"} 2024-12-10T01:16:54,030 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T01:16:54,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T01:16:54,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-10T01:16:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-12-10T01:16:55,011 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:55,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37aadc7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,014 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c3b0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407cb20c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:55,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:55,023 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T01:16:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fd1a2b66caf8,37249,1733793261894 2024-12-10T01:16:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5c75fc64 2024-12-10T01:16:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T01:16:55,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T01:16:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T01:16:55,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T01:16:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T01:16:55,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742292_1468 (size=1097) 2024-12-10T01:16:55,443 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T01:16:55,443 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T01:16:55,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:16:55,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE}] 2024-12-10T01:16:55,452 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE 2024-12-10T01:16:55,453 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:55,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE because future has completed 2024-12-10T01:16:55,455 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:16:55,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:55,609 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,610 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:16:55,610 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing bff4d84e434d4670f70bf8a0a3136faf, disabling compactions & flushes 2024-12-10T01:16:55,610 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,610 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,610 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. after waiting 0 ms 2024-12-10T01:16:55,610 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,620 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T01:16:55,621 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,621 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for bff4d84e434d4670f70bf8a0a3136faf: Waiting for close lock at 1733793415610Running coprocessor pre-close hooks at 1733793415610Disabling compacts and flushes for region at 1733793415610Disabling writes for close at 1733793415610Writing region close event to WAL at 1733793415611 (+1 ms)Running coprocessor post-close hooks at 1733793415621 (+10 ms)Closed at 1733793415621 2024-12-10T01:16:55,621 WARN [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionServer(3299): Not adding moved region record: bff4d84e434d4670f70bf8a0a3136faf to self. 2024-12-10T01:16:55,623 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=CLOSED 2024-12-10T01:16:55,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:55,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-10T01:16:55,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 in 171 msec 2024-12-10T01:16:55,629 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE; state=CLOSED, location=fd1a2b66caf8,38925,1733793262612; forceNewPlan=false, retain=true 2024-12-10T01:16:55,780 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=OPENING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:55,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE because future has completed 2024-12-10T01:16:55,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:16:55,945 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,946 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(7752): Opening region: {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} 2024-12-10T01:16:55,946 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,947 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T01:16:55,947 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(7794): checking encryption for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,947 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(7797): checking classloading for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,950 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,952 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:55,952 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName A 2024-12-10T01:16:55,953 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:55,954 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:55,954 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,954 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:55,955 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName B 2024-12-10T01:16:55,955 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:55,955 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:55,955 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,955 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T01:16:55,955 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bff4d84e434d4670f70bf8a0a3136faf columnFamilyName C 2024-12-10T01:16:55,956 DEBUG [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:55,956 INFO [StoreOpener-bff4d84e434d4670f70bf8a0a3136faf-1 {}] regionserver.HStore(327): Store=bff4d84e434d4670f70bf8a0a3136faf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T01:16:55,956 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,956 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1038): replaying wal for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,956 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,957 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,958 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1048): stopping wal replay for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,958 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1060): Cleaning up temporary data for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,958 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T01:16:55,959 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1093): writing seq id for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,960 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1114): Opened bff4d84e434d4670f70bf8a0a3136faf; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62592408, jitterRate=-0.06730043888092041}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T01:16:55,960 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:55,960 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegion(1006): Region open journal for bff4d84e434d4670f70bf8a0a3136faf: Running coprocessor pre-open hook at 1733793415947Writing region info on filesystem at 1733793415947Initializing all the Stores at 1733793415949 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793415949Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793415950 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733793415950Cleaning up temporary data from old regions at 1733793415958 (+8 ms)Running coprocessor post-open hooks at 1733793415960 (+2 ms)Region opened successfully at 1733793415960 2024-12-10T01:16:55,961 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., pid=129, masterSystemTime=1733793415939 2024-12-10T01:16:55,962 DEBUG [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,962 INFO [RS_OPEN_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_OPEN_REGION, pid=129}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:55,962 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=OPEN, openSeqNum=5, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:55,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:16:55,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=127 2024-12-10T01:16:55,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 in 180 msec 2024-12-10T01:16:55,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-12-10T01:16:55,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, REOPEN/MOVE in 515 msec 2024-12-10T01:16:55,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T01:16:55,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-12-10T01:16:55,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-12-10T01:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T01:16:55,972 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:16:55,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e09ae99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,974 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfb7cbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,975 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d19c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500aa36d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d15779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,978 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f168d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8ecbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11dc87c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c12539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3c452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bcb16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fa6826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5500ca1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,984 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bea73dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13798f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdd7f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55be6699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb7f18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba1ca28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6a0992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4f30b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbf29c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7484e216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275bf375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aacc33c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13714235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1d2af5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e14e647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd1a2b66caf8,37249,-1 for getting cluster id 2024-12-10T01:16:55,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T01:16:55,998 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '97129feb-7bb1-48e1-a470-4d1c86822c6e' 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "97129feb-7bb1-48e1-a470-4d1c86822c6e" 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e48cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd1a2b66caf8,37249,-1] 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T01:16:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:16:55,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T01:16:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12974c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T01:16:56,000 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,000 DEBUG [Thread-2120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,000 DEBUG [Thread-2118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,000 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,000 DEBUG [Thread-2135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,000 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [Thread-2122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:16:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,003 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,003 DEBUG [Thread-2124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,003 DEBUG [Thread-2129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T01:16:56,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd1a2b66caf8,38925,1733793262612, seqNum=-1] 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees 2024-12-10T01:16:56,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T01:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:56,008 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:16:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5] 2024-12-10T01:16:56,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:16:56,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:56,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T01:16:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:16:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:16:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:16:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793476027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793476029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793476029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793476030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793476030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121021cd5ae268a54e66a46ac2cd663128c5_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793416008/Put/seqid=0 2024-12-10T01:16:56,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742293_1469 (size=12154) 2024-12-10T01:16:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:56,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793476139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793476139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793476139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793476139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793476139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793476350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793476350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793476350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793476350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793476350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:56,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121021cd5ae268a54e66a46ac2cd663128c5_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121021cd5ae268a54e66a46ac2cd663128c5_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:56,454 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8c40648293d747d2b1ecf942ce5d7858, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:16:56,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8c40648293d747d2b1ecf942ce5d7858 is 175, key is test_row_0/A:col10/1733793416008/Put/seqid=0 2024-12-10T01:16:56,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742294_1470 (size=30955) 2024-12-10T01:16:56,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793476660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793476660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793476660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793476660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793476660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:56,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8c40648293d747d2b1ecf942ce5d7858 2024-12-10T01:16:56,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/b139e94efdbe44f480ef91dae731265d is 50, key is test_row_0/B:col10/1733793416008/Put/seqid=0 2024-12-10T01:16:56,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742295_1471 (size=12001) 2024-12-10T01:16:56,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:56,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793477169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793477170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793477170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793477170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793477170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:57,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:57,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/b139e94efdbe44f480ef91dae731265d 2024-12-10T01:16:57,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/d3d602c73bfd4dd3ad73584d5151fb17 is 50, key is test_row_0/C:col10/1733793416008/Put/seqid=0 2024-12-10T01:16:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742296_1472 (size=12001) 2024-12-10T01:16:57,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,506 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T01:16:57,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:57,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:16:57,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/d3d602c73bfd4dd3ad73584d5151fb17 2024-12-10T01:16:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8c40648293d747d2b1ecf942ce5d7858 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858 2024-12-10T01:16:57,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858, entries=150, sequenceid=16, filesize=30.2 K 2024-12-10T01:16:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/b139e94efdbe44f480ef91dae731265d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d 2024-12-10T01:16:57,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T01:16:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/d3d602c73bfd4dd3ad73584d5151fb17 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17 2024-12-10T01:16:57,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T01:16:57,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bff4d84e434d4670f70bf8a0a3136faf in 1706ms, sequenceid=16, compaction requested=false 2024-12-10T01:16:57,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:16:57,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T01:16:57,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:57,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:16:57,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:16:57,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d1b008c9ddb84a63829ca50a5c4031de_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793416028/Put/seqid=0 2024-12-10T01:16:57,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742297_1473 (size=12154) 2024-12-10T01:16:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:16:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:16:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793478189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793478190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,193 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_2' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:16:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793478191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793478193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793478193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:16:58,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d1b008c9ddb84a63829ca50a5c4031de_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d1b008c9ddb84a63829ca50a5c4031de_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:16:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6cf3fcfb69948dea036e5ca9433403a, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:16:58,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6cf3fcfb69948dea036e5ca9433403a is 175, key is test_row_0/A:col10/1733793416028/Put/seqid=0 2024-12-10T01:16:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742298_1474 (size=30955) 2024-12-10T01:16:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793478299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793478299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793478299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793478299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793478509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793478509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793478509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793478509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6cf3fcfb69948dea036e5ca9433403a 2024-12-10T01:16:58,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/941c5722c0f14883a87368506781c06e is 50, key is test_row_0/B:col10/1733793416028/Put/seqid=0 2024-12-10T01:16:58,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742299_1475 (size=12001) 2024-12-10T01:16:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793478820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793478820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793478820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793478820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:59,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/941c5722c0f14883a87368506781c06e 2024-12-10T01:16:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/089222c3780942aeb2e793648cc16783 is 50, key is test_row_0/C:col10/1733793416028/Put/seqid=0 2024-12-10T01:16:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742300_1476 (size=12001) 2024-12-10T01:16:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793479330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793479330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793479330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:59,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:16:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793479339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:16:59,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:16:59,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:16:59,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/089222c3780942aeb2e793648cc16783 2024-12-10T01:16:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6cf3fcfb69948dea036e5ca9433403a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a 2024-12-10T01:16:59,479 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a, entries=150, sequenceid=36, filesize=30.2 K 2024-12-10T01:16:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/941c5722c0f14883a87368506781c06e as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e 2024-12-10T01:16:59,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T01:16:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/089222c3780942aeb2e793648cc16783 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783 2024-12-10T01:16:59,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T01:16:59,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bff4d84e434d4670f70bf8a0a3136faf in 1647ms, sequenceid=36, compaction requested=false 2024-12-10T01:16:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:16:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:16:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-12-10T01:16:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=131 2024-12-10T01:16:59,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-10T01:16:59,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4770 sec 2024-12-10T01:16:59,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees in 3.4870 sec 2024-12-10T01:17:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-10T01:17:00,160 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:00,160 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-12-10T01:17:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:00,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:00,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:00,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:00,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:00,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106fa3271b9b504b1c99e6e51ecd5b88de_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:00,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742301_1477 (size=14594) 2024-12-10T01:17:00,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793480265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:00,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:00,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:00,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793480339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,341 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793480349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793480349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793480349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,351 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:00,351 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,351 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793480379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:00,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793480589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:00,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106fa3271b9b504b1c99e6e51ecd5b88de_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106fa3271b9b504b1c99e6e51ecd5b88de_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:00,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b1596bf46da5423bbb0ad1386aeb2bc1, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:00,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b1596bf46da5423bbb0ad1386aeb2bc1 is 175, key is test_row_0/A:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:00,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742302_1478 (size=39549) 2024-12-10T01:17:00,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:00,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793480900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:00,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:00,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,036 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b1596bf46da5423bbb0ad1386aeb2bc1 2024-12-10T01:17:01,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/3c1e99dbc5804b60a2a7730b61296cb5 is 50, key is test_row_0/B:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742303_1479 (size=12001) 2024-12-10T01:17:01,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:01,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:01,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793481409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:01,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/3c1e99dbc5804b60a2a7730b61296cb5 2024-12-10T01:17:01,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/970198c69131421e8ca81a240492e1a9 is 50, key is test_row_0/C:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742304_1480 (size=12001) 2024-12-10T01:17:01,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/970198c69131421e8ca81a240492e1a9 2024-12-10T01:17:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b1596bf46da5423bbb0ad1386aeb2bc1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1 2024-12-10T01:17:01,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1, entries=200, sequenceid=49, filesize=38.6 K 2024-12-10T01:17:01,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/3c1e99dbc5804b60a2a7730b61296cb5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5 2024-12-10T01:17:01,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5, entries=150, sequenceid=49, filesize=11.7 K 2024-12-10T01:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/970198c69131421e8ca81a240492e1a9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9 2024-12-10T01:17:01,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9, entries=150, sequenceid=49, filesize=11.7 K 2024-12-10T01:17:01,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for bff4d84e434d4670f70bf8a0a3136faf in 1262ms, sequenceid=49, compaction requested=true 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:01,473 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:01,473 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:01,474 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,474 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,474 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=99.1 K 2024-12-10T01:17:01,474 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.2 K 2024-12-10T01:17:01,474 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1] 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b139e94efdbe44f480ef91dae731265d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733793416008 2024-12-10T01:17:01,474 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c40648293d747d2b1ecf942ce5d7858, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733793416008 2024-12-10T01:17:01,475 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 941c5722c0f14883a87368506781c06e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733793416027 2024-12-10T01:17:01,475 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cf3fcfb69948dea036e5ca9433403a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733793416027 2024-12-10T01:17:01,475 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1e99dbc5804b60a2a7730b61296cb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:01,475 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1596bf46da5423bbb0ad1386aeb2bc1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:01,482 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:01,483 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:01,483 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121030a4c3a1e2c041009760c0b6408eafd3_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:01,483 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/ec5ad2cbefde4967b19c79f7fe53e5ba is 50, key is test_row_0/B:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:01,485 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121030a4c3a1e2c041009760c0b6408eafd3_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:01,485 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121030a4c3a1e2c041009760c0b6408eafd3_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:01,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742305_1481 (size=12104) 2024-12-10T01:17:01,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742306_1482 (size=4469) 2024-12-10T01:17:01,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T01:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,534 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:01,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121008d3033ece294b46924edada838f1832_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793420260/Put/seqid=0 2024-12-10T01:17:01,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742307_1483 (size=12154) 2024-12-10T01:17:01,890 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#409 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:01,890 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/3da34426c7e04940a851a2eb09aebe6a is 175, key is test_row_0/A:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:01,892 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/ec5ad2cbefde4967b19c79f7fe53e5ba as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/ec5ad2cbefde4967b19c79f7fe53e5ba 2024-12-10T01:17:01,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742308_1484 (size=31058) 2024-12-10T01:17:01,896 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into ec5ad2cbefde4967b19c79f7fe53e5ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:01,896 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:01,896 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793421473; duration=0sec 2024-12-10T01:17:01,897 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:01,897 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:01,897 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:01,898 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/3da34426c7e04940a851a2eb09aebe6a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a 2024-12-10T01:17:01,898 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:01,898 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:01,898 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:01,898 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.2 K 2024-12-10T01:17:01,898 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d602c73bfd4dd3ad73584d5151fb17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733793416008 2024-12-10T01:17:01,899 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 089222c3780942aeb2e793648cc16783, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733793416027 2024-12-10T01:17:01,899 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 970198c69131421e8ca81a240492e1a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:01,901 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into 3da34426c7e04940a851a2eb09aebe6a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:01,901 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:01,901 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793421473; duration=0sec 2024-12-10T01:17:01,901 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:01,901 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:01,904 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:01,904 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/d63640b5e6334f8dbb675a4e0ee63c76 is 50, key is test_row_0/C:col10/1733793418185/Put/seqid=0 2024-12-10T01:17:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742309_1485 (size=12104) 2024-12-10T01:17:01,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:01,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121008d3033ece294b46924edada838f1832_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121008d3033ece294b46924edada838f1832_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:01,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/85311e4662ee4756b9d2f76eaa1980aa, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:01,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/85311e4662ee4756b9d2f76eaa1980aa is 175, key is test_row_0/A:col10/1733793420260/Put/seqid=0 2024-12-10T01:17:01,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742310_1486 (size=30955) 2024-12-10T01:17:02,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T01:17:02,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T01:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:02,310 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/d63640b5e6334f8dbb675a4e0ee63c76 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d63640b5e6334f8dbb675a4e0ee63c76 2024-12-10T01:17:02,313 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into d63640b5e6334f8dbb675a4e0ee63c76(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:02,313 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:02,314 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793421473; duration=0sec 2024-12-10T01:17:02,314 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:02,314 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:02,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/85311e4662ee4756b9d2f76eaa1980aa 2024-12-10T01:17:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c3334cec09ca4909a829ce252c722b89 is 50, key is test_row_0/B:col10/1733793420260/Put/seqid=0 2024-12-10T01:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:02,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:02,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742311_1487 (size=12001) 2024-12-10T01:17:02,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793482374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793482375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793482377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793482379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,380 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793482420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,421 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793482479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793482479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793482489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793482689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793482689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793482699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:02,763 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c3334cec09ca4909a829ce252c722b89 2024-12-10T01:17:02,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af9e58599219401d9576ac9b6696d64c is 50, key is test_row_0/C:col10/1733793420260/Put/seqid=0 2024-12-10T01:17:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742312_1488 (size=12001) 2024-12-10T01:17:03,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793483000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793483000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793483009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,171 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af9e58599219401d9576ac9b6696d64c 2024-12-10T01:17:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/85311e4662ee4756b9d2f76eaa1980aa as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa 2024-12-10T01:17:03,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa, entries=150, sequenceid=68, filesize=30.2 K 2024-12-10T01:17:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c3334cec09ca4909a829ce252c722b89 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89 2024-12-10T01:17:03,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T01:17:03,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af9e58599219401d9576ac9b6696d64c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c 2024-12-10T01:17:03,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T01:17:03,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for bff4d84e434d4670f70bf8a0a3136faf in 1652ms, sequenceid=68, compaction requested=false 2024-12-10T01:17:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:03,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-12-10T01:17:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=133 2024-12-10T01:17:03,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-10T01:17:03,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0250 sec 2024-12-10T01:17:03,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 3.0290 sec 2024-12-10T01:17:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:03,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:03,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f3b9a018fa434aa0868cd3e0f4955a33_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:03,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742313_1489 (size=17034) 2024-12-10T01:17:03,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793483534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793483540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793483540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793483649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793483649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793483649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793483859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793483859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793483860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:03,921 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:03,924 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f3b9a018fa434aa0868cd3e0f4955a33_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3b9a018fa434aa0868cd3e0f4955a33_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:03,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bc7cd21158b64f4b8d9e095bc1a63831, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:03,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bc7cd21158b64f4b8d9e095bc1a63831 is 175, key is test_row_0/A:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:03,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742314_1490 (size=48139) 2024-12-10T01:17:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793484169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793484169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793484170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T01:17:04,319 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:04,319 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-12-10T01:17:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:04,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:04,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:04,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:04,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bc7cd21158b64f4b8d9e095bc1a63831 2024-12-10T01:17:04,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d1ad89e808fb4c31bf41cda1bd5aac08 is 50, key is test_row_0/B:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742315_1491 (size=12001) 2024-12-10T01:17:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793484439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,440 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T01:17:04,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:04,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T01:17:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793484680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793484680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793484680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:04,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d1ad89e808fb4c31bf41cda1bd5aac08 2024-12-10T01:17:04,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/33e1706f29994b4d83672e3cf7a99dc6 is 50, key is test_row_0/C:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:04,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742316_1492 (size=12001) 2024-12-10T01:17:04,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/33e1706f29994b4d83672e3cf7a99dc6 2024-12-10T01:17:04,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bc7cd21158b64f4b8d9e095bc1a63831 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831 2024-12-10T01:17:04,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831, entries=250, sequenceid=86, filesize=47.0 K 2024-12-10T01:17:04,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d1ad89e808fb4c31bf41cda1bd5aac08 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08 2024-12-10T01:17:04,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08, entries=150, sequenceid=86, filesize=11.7 K 2024-12-10T01:17:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/33e1706f29994b4d83672e3cf7a99dc6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6 2024-12-10T01:17:04,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6, entries=150, sequenceid=86, filesize=11.7 K 2024-12-10T01:17:04,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for bff4d84e434d4670f70bf8a0a3136faf in 1253ms, sequenceid=86, compaction requested=true 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:04,765 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:04,765 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:04,766 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,766 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,766 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/ec5ad2cbefde4967b19c79f7fe53e5ba, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.3 K 2024-12-10T01:17:04,766 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=107.6 K 2024-12-10T01:17:04,766 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831] 2024-12-10T01:17:04,766 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5ad2cbefde4967b19c79f7fe53e5ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:04,767 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3da34426c7e04940a851a2eb09aebe6a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:04,767 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c3334cec09ca4909a829ce252c722b89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733793420234 2024-12-10T01:17:04,767 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ad89e808fb4c31bf41cda1bd5aac08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422376 2024-12-10T01:17:04,767 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85311e4662ee4756b9d2f76eaa1980aa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733793420234 2024-12-10T01:17:04,767 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7cd21158b64f4b8d9e095bc1a63831, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422372 2024-12-10T01:17:04,772 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:04,772 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:04,772 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/f5f330cbc80a4319a65c96f8b8c912d5 is 50, key is test_row_0/B:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:04,773 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103fc92f775a7f4ba5a433ff4129d1d614_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:04,774 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103fc92f775a7f4ba5a433ff4129d1d614_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:04,775 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103fc92f775a7f4ba5a433ff4129d1d614_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:04,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742317_1493 (size=12207) 2024-12-10T01:17:04,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,778 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742318_1494 (size=4469) 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:04,780 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#418 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:04,780 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0700ed90d58848d8acfa9354517785a5 is 175, key is test_row_0/A:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:04,781 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/f5f330cbc80a4319a65c96f8b8c912d5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/f5f330cbc80a4319a65c96f8b8c912d5 2024-12-10T01:17:04,784 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into f5f330cbc80a4319a65c96f8b8c912d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:04,784 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:04,785 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793424765; duration=0sec 2024-12-10T01:17:04,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:04,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:04,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:04,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:04,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:04,785 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:04,786 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d63640b5e6334f8dbb675a4e0ee63c76, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.3 K 2024-12-10T01:17:04,786 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d63640b5e6334f8dbb675a4e0ee63c76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733793418185 2024-12-10T01:17:04,786 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting af9e58599219401d9576ac9b6696d64c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733793420234 2024-12-10T01:17:04,786 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e1706f29994b4d83672e3cf7a99dc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422376 2024-12-10T01:17:04,794 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:04,795 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/2dcb18d5f7cd40bb919ed0a01274a0f1 is 50, key is test_row_0/C:col10/1733793423511/Put/seqid=0 2024-12-10T01:17:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210085bad8de9ff402eba1a1fb22ad7aa04_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793423536/Put/seqid=0 2024-12-10T01:17:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742319_1495 (size=31161) 2024-12-10T01:17:04,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742320_1496 (size=12207) 2024-12-10T01:17:04,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742321_1497 (size=12154) 2024-12-10T01:17:04,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:04,806 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210085bad8de9ff402eba1a1fb22ad7aa04_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210085bad8de9ff402eba1a1fb22ad7aa04_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:04,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/fbf6194a711342809de42d45ada46e11, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:04,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/fbf6194a711342809de42d45ada46e11 is 175, key is test_row_0/A:col10/1733793423536/Put/seqid=0 2024-12-10T01:17:04,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742322_1498 (size=30955) 2024-12-10T01:17:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:05,212 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/fbf6194a711342809de42d45ada46e11 2024-12-10T01:17:05,247 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0700ed90d58848d8acfa9354517785a5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5 2024-12-10T01:17:05,248 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/2dcb18d5f7cd40bb919ed0a01274a0f1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/2dcb18d5f7cd40bb919ed0a01274a0f1 2024-12-10T01:17:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/96b8ef0a062342a4a0b12db669445b96 is 50, key is test_row_0/B:col10/1733793423536/Put/seqid=0 2024-12-10T01:17:05,253 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into 0700ed90d58848d8acfa9354517785a5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:05,253 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:05,253 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793424765; duration=0sec 2024-12-10T01:17:05,253 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:05,253 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:05,254 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into 2dcb18d5f7cd40bb919ed0a01274a0f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:05,254 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:05,254 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793424765; duration=0sec 2024-12-10T01:17:05,254 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:05,254 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:05,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742323_1499 (size=12001) 2024-12-10T01:17:05,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/96b8ef0a062342a4a0b12db669445b96 2024-12-10T01:17:05,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/8181c4c73b634cd798426f0a28153bcf is 50, key is test_row_0/C:col10/1733793423536/Put/seqid=0 2024-12-10T01:17:05,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742324_1500 (size=12001) 2024-12-10T01:17:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:05,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/8181c4c73b634cd798426f0a28153bcf 2024-12-10T01:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/fbf6194a711342809de42d45ada46e11 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11 2024-12-10T01:17:05,677 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11, entries=150, sequenceid=104, filesize=30.2 K 2024-12-10T01:17:05,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/96b8ef0a062342a4a0b12db669445b96 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96 2024-12-10T01:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96, entries=150, sequenceid=104, filesize=11.7 K 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/8181c4c73b634cd798426f0a28153bcf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf, entries=150, sequenceid=104, filesize=11.7 K 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for bff4d84e434d4670f70bf8a0a3136faf in 906ms, sequenceid=104, compaction requested=false 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.3680 sec 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:05,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:05,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d08321697b614e5d8a6f6f1ec70d47e6_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:05,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742325_1501 (size=21918) 2024-12-10T01:17:05,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:05,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d08321697b614e5d8a6f6f1ec70d47e6_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d08321697b614e5d8a6f6f1ec70d47e6_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:05,775 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/205ed856ea3547c29ffb29d1497e9438, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:05,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/205ed856ea3547c29ffb29d1497e9438 is 175, key is test_row_0/A:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793425753, max timestamp=1733793425778Num uniques 859; 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793425753, max timestamp=1733793425778Num uniques 859; 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733793425753, max timestamp=1733793425778Num uniques 859; 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:17:05,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd1a2b66caf8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T01:17:05,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742326_1502 (size=65323) 2024-12-10T01:17:05,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/205ed856ea3547c29ffb29d1497e9438 2024-12-10T01:17:05,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/fc7f1a65549144d08f12feea6ffe9e8a is 50, key is test_row_0/B:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:05,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742327_1503 (size=12001) 2024-12-10T01:17:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793485803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793485803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793485807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793485909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793485909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:05,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793485924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793486119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793486119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793486130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/fc7f1a65549144d08f12feea6ffe9e8a 2024-12-10T01:17:06,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/82f97a712800415bb9e589d2f2c91b35 is 50, key is test_row_0/C:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:06,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742328_1504 (size=12001) 2024-12-10T01:17:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793486429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793486429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793486429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,431 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793486439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T01:17:06,470 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:06,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-12-10T01:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:06,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:06,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:06,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:06,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/82f97a712800415bb9e589d2f2c91b35 2024-12-10T01:17:06,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/205ed856ea3547c29ffb29d1497e9438 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438 2024-12-10T01:17:06,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438, entries=350, sequenceid=117, filesize=63.8 K 2024-12-10T01:17:06,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/fc7f1a65549144d08f12feea6ffe9e8a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a 2024-12-10T01:17:06,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T01:17:06,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T01:17:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/82f97a712800415bb9e589d2f2c91b35 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35 2024-12-10T01:17:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:06,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T01:17:06,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bff4d84e434d4670f70bf8a0a3136faf in 887ms, sequenceid=117, compaction requested=true 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:06,627 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:06,627 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:06,628 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,628 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/f5f330cbc80a4319a65c96f8b8c912d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.4 K 2024-12-10T01:17:06,628 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=124.5 K 2024-12-10T01:17:06,628 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438] 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f330cbc80a4319a65c96f8b8c912d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422376 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b8ef0a062342a4a0b12db669445b96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733793423532 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0700ed90d58848d8acfa9354517785a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422376 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7f1a65549144d08f12feea6ffe9e8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:06,628 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf6194a711342809de42d45ada46e11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733793423532 2024-12-10T01:17:06,629 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 205ed856ea3547c29ffb29d1497e9438, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:06,633 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:06,633 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:06,634 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/53e92106ca2a4bc29e0d4979f4d5c542 is 50, key is test_row_0/B:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:06,635 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412102ffb0b21751f4f588720a25a902a18c8_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:06,636 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412102ffb0b21751f4f588720a25a902a18c8_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:06,637 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102ffb0b21751f4f588720a25a902a18c8_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:06,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742330_1506 (size=4469) 2024-12-10T01:17:06,640 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#427 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:06,641 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/ff5624d6e4084d9ca730c3a82e200bab is 175, key is test_row_0/A:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:06,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742331_1507 (size=31263) 2024-12-10T01:17:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742329_1505 (size=12309) 2024-12-10T01:17:06,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:06,776 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T01:17:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d560048c7fa94840af2517897d021eb2_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793425801/Put/seqid=0 2024-12-10T01:17:06,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742332_1508 (size=12304) 2024-12-10T01:17:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:06,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d560048c7fa94840af2517897d021eb2_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d560048c7fa94840af2517897d021eb2_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/e139d3dda0d947fdbfb6d48ebc7702d8, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/e139d3dda0d947fdbfb6d48ebc7702d8 is 175, key is test_row_0/A:col10/1733793425801/Put/seqid=0 2024-12-10T01:17:06,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742333_1509 (size=31105) 2024-12-10T01:17:06,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793486951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793486954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:06,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793486954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,048 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/ff5624d6e4084d9ca730c3a82e200bab as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab 2024-12-10T01:17:07,051 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into ff5624d6e4084d9ca730c3a82e200bab(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:07,051 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:07,051 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793426627; duration=0sec 2024-12-10T01:17:07,051 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:07,051 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:07,051 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:07,052 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:07,052 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:07,052 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:07,052 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/2dcb18d5f7cd40bb919ed0a01274a0f1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.4 K 2024-12-10T01:17:07,052 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dcb18d5f7cd40bb919ed0a01274a0f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733793422376 2024-12-10T01:17:07,053 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8181c4c73b634cd798426f0a28153bcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733793423532 2024-12-10T01:17:07,053 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f97a712800415bb9e589d2f2c91b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:07,057 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:07,057 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/f70c3d14b10748c0a1271a7764041750 is 50, key is test_row_0/C:col10/1733793425739/Put/seqid=0 2024-12-10T01:17:07,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742334_1510 (size=12309) 2024-12-10T01:17:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793487060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793487059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,065 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/53e92106ca2a4bc29e0d4979f4d5c542 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/53e92106ca2a4bc29e0d4979f4d5c542 2024-12-10T01:17:07,068 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into 53e92106ca2a4bc29e0d4979f4d5c542(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:07,068 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:07,068 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793426627; duration=0sec 2024-12-10T01:17:07,068 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:07,068 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:07,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793487069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:07,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/e139d3dda0d947fdbfb6d48ebc7702d8 2024-12-10T01:17:07,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/15a6ae9139bb476e92051ebb189d190b is 50, key is test_row_0/B:col10/1733793425801/Put/seqid=0 2024-12-10T01:17:07,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742335_1511 (size=12151) 2024-12-10T01:17:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793487269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793487270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793487280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,463 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/f70c3d14b10748c0a1271a7764041750 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f70c3d14b10748c0a1271a7764041750 2024-12-10T01:17:07,467 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into f70c3d14b10748c0a1271a7764041750(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:07,467 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:07,467 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793426627; duration=0sec 2024-12-10T01:17:07,467 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:07,467 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:07,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793487579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793487579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793487589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:07,611 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/15a6ae9139bb476e92051ebb189d190b 2024-12-10T01:17:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/e347885ace8c4ea3a15205308d505ac1 is 50, key is test_row_0/C:col10/1733793425801/Put/seqid=0 2024-12-10T01:17:07,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742336_1512 (size=12151) 2024-12-10T01:17:08,022 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/e347885ace8c4ea3a15205308d505ac1 2024-12-10T01:17:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/e139d3dda0d947fdbfb6d48ebc7702d8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8 2024-12-10T01:17:08,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8, entries=150, sequenceid=153, filesize=30.4 K 2024-12-10T01:17:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/15a6ae9139bb476e92051ebb189d190b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b 2024-12-10T01:17:08,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b, entries=150, sequenceid=153, filesize=11.9 K 2024-12-10T01:17:08,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/e347885ace8c4ea3a15205308d505ac1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1 2024-12-10T01:17:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1, entries=150, sequenceid=153, filesize=11.9 K 2024-12-10T01:17:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for bff4d84e434d4670f70bf8a0a3136faf in 1258ms, sequenceid=153, compaction requested=false 2024-12-10T01:17:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-12-10T01:17:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-12-10T01:17:08,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-10T01:17:08,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-12-10T01:17:08,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 1.5670 sec 2024-12-10T01:17:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:08,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:08,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:08,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107af77f65e5a34de799f2a19cb804b73e_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:08,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742337_1513 (size=17284) 2024-12-10T01:17:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793488145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793488150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793488150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793488260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793488260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793488260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793488470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793488470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793488470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1733793488479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,480 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_1' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:08,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107af77f65e5a34de799f2a19cb804b73e_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107af77f65e5a34de799f2a19cb804b73e_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:08,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b3220c5843284b15b5a0d469c307435b, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:08,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b3220c5843284b15b5a0d469c307435b is 175, key is test_row_0/A:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742338_1514 (size=48389) 2024-12-10T01:17:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T01:17:08,629 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:08,630 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-12-10T01:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:08,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:08,632 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:08,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:08,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793488779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793488780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793488780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:08,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T01:17:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:08,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b3220c5843284b15b5a0d469c307435b 2024-12-10T01:17:08,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c2f264e8d80b4f5a99123563b3f408cb is 50, key is test_row_0/B:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:08,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742339_1515 (size=12151) 2024-12-10T01:17:08,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c2f264e8d80b4f5a99123563b3f408cb 2024-12-10T01:17:08,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c48c3e7df6324519b3963840816cb0e1 is 50, key is test_row_0/C:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:08,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742340_1516 (size=12151) 2024-12-10T01:17:08,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T01:17:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:08,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:09,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T01:17:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T01:17:09,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:09,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793489290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793489290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793489290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:09,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c48c3e7df6324519b3963840816cb0e1 2024-12-10T01:17:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/b3220c5843284b15b5a0d469c307435b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b 2024-12-10T01:17:09,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b, entries=250, sequenceid=169, filesize=47.3 K 2024-12-10T01:17:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/c2f264e8d80b4f5a99123563b3f408cb as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb 2024-12-10T01:17:09,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T01:17:09,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c48c3e7df6324519b3963840816cb0e1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1 2024-12-10T01:17:09,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T01:17:09,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for bff4d84e434d4670f70bf8a0a3136faf in 1249ms, sequenceid=169, compaction requested=true 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:09,341 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:09,341 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:09,342 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,342 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,342 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=108.2 K 2024-12-10T01:17:09,342 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/53e92106ca2a4bc29e0d4979f4d5c542, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.8 K 2024-12-10T01:17:09,342 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b] 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff5624d6e4084d9ca730c3a82e200bab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:09,342 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e92106ca2a4bc29e0d4979f4d5c542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:09,343 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a6ae9139bb476e92051ebb189d190b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733793425801 2024-12-10T01:17:09,343 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e139d3dda0d947fdbfb6d48ebc7702d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733793425801 2024-12-10T01:17:09,343 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3220c5843284b15b5a0d469c307435b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:09,343 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f264e8d80b4f5a99123563b3f408cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:09,348 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:09,348 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/2d6978ed799240f3add6961de324240f is 50, key is test_row_0/B:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:09,349 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:09,357 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121074a907da679e42f4b95bf701ef266cea_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:09,359 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121074a907da679e42f4b95bf701ef266cea_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:09,359 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121074a907da679e42f4b95bf701ef266cea_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:09,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742341_1517 (size=12561) 2024-12-10T01:17:09,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742342_1518 (size=4469) 2024-12-10T01:17:09,379 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#436 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:09,379 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/ab1c340dd8d548efa1c8c59f32dba6f2 is 175, key is test_row_0/A:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:09,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742343_1519 (size=31515) 2024-12-10T01:17:09,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T01:17:09,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,393 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101e5c49f427b34d1bb62ab36af7647fbb_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793428148/Put/seqid=0 2024-12-10T01:17:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742344_1520 (size=12304) 2024-12-10T01:17:09,769 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/2d6978ed799240f3add6961de324240f as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/2d6978ed799240f3add6961de324240f 2024-12-10T01:17:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:09,772 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into 2d6978ed799240f3add6961de324240f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:09,772 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:09,772 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793429341; duration=0sec 2024-12-10T01:17:09,772 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:09,772 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:09,772 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:09,773 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:09,773 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:09,773 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:09,773 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f70c3d14b10748c0a1271a7764041750, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=35.8 K 2024-12-10T01:17:09,773 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting f70c3d14b10748c0a1271a7764041750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733793425719 2024-12-10T01:17:09,774 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e347885ace8c4ea3a15205308d505ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733793425801 2024-12-10T01:17:09,774 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c48c3e7df6324519b3963840816cb0e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:09,778 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:09,778 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/b6c8c1e02a38417885db711abe4566c5 is 50, key is test_row_0/C:col10/1733793426949/Put/seqid=0 2024-12-10T01:17:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742345_1521 (size=12561) 2024-12-10T01:17:09,785 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/b6c8c1e02a38417885db711abe4566c5 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b6c8c1e02a38417885db711abe4566c5 2024-12-10T01:17:09,788 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/ab1c340dd8d548efa1c8c59f32dba6f2 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2 2024-12-10T01:17:09,789 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into b6c8c1e02a38417885db711abe4566c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:09,790 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793429341; duration=0sec 2024-12-10T01:17:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:09,790 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:09,791 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into ab1c340dd8d548efa1c8c59f32dba6f2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:09,791 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:09,791 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793429341; duration=0sec 2024-12-10T01:17:09,791 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:09,791 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:09,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:09,814 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101e5c49f427b34d1bb62ab36af7647fbb_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101e5c49f427b34d1bb62ab36af7647fbb_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bace80c0e7654cd09951430f9779482d, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:09,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bace80c0e7654cd09951430f9779482d is 175, key is test_row_0/A:col10/1733793428148/Put/seqid=0 2024-12-10T01:17:09,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742346_1522 (size=31105) 2024-12-10T01:17:10,219 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bace80c0e7654cd09951430f9779482d 2024-12-10T01:17:10,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/e236c89a4dd84dda8789bc09f640928b is 50, key is test_row_0/B:col10/1733793428148/Put/seqid=0 2024-12-10T01:17:10,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742347_1523 (size=12151) 2024-12-10T01:17:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:10,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793490318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793490319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793490319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793490429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793490429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793490430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793490639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793490639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793490639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/e236c89a4dd84dda8789bc09f640928b 2024-12-10T01:17:10,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/b5288246a5814bf5ae70ccbe28df6769 is 50, key is test_row_0/C:col10/1733793428148/Put/seqid=0 2024-12-10T01:17:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742348_1524 (size=12151) 2024-12-10T01:17:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793490950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793490950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793490950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/b5288246a5814bf5ae70ccbe28df6769 2024-12-10T01:17:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/bace80c0e7654cd09951430f9779482d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d 2024-12-10T01:17:11,098 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d, entries=150, sequenceid=189, filesize=30.4 K 2024-12-10T01:17:11,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/e236c89a4dd84dda8789bc09f640928b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b 2024-12-10T01:17:11,101 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b, entries=150, sequenceid=189, filesize=11.9 K 2024-12-10T01:17:11,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/b5288246a5814bf5ae70ccbe28df6769 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769 2024-12-10T01:17:11,104 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769, entries=150, sequenceid=189, filesize=11.9 K 2024-12-10T01:17:11,105 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bff4d84e434d4670f70bf8a0a3136faf in 1712ms, sequenceid=189, compaction requested=false 2024-12-10T01:17:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-12-10T01:17:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-12-10T01:17:11,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-10T01:17:11,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-12-10T01:17:11,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 2.4780 sec 2024-12-10T01:17:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:11,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:11,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121066ae2517d3c94fa4a5d9fb8c353fcf47_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:11,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742349_1525 (size=17284) 2024-12-10T01:17:11,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:11,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793491480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121066ae2517d3c94fa4a5d9fb8c353fcf47_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121066ae2517d3c94fa4a5d9fb8c353fcf47_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:11,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0ff7f0330dc34033ab9ce08aaa087066, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793491482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793491481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0ff7f0330dc34033ab9ce08aaa087066 is 175, key is test_row_0/A:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742350_1526 (size=48389) 2024-12-10T01:17:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793491589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793491589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793491589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793491799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793491800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793491800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:11,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0ff7f0330dc34033ab9ce08aaa087066 2024-12-10T01:17:11,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/5b7b14e6e2cd4360905a32cfe56662b3 is 50, key is test_row_0/B:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742351_1527 (size=12151) 2024-12-10T01:17:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793492109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793492109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793492109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/5b7b14e6e2cd4360905a32cfe56662b3 2024-12-10T01:17:12,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/deb95d21bba343d99b6fc54e34d8ebb6 is 50, key is test_row_0/C:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:12,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742352_1528 (size=12151) 2024-12-10T01:17:12,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793492619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793492619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793492619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:12,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/deb95d21bba343d99b6fc54e34d8ebb6 2024-12-10T01:17:12,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/0ff7f0330dc34033ab9ce08aaa087066 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066 2024-12-10T01:17:12,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066, entries=250, sequenceid=207, filesize=47.3 K 2024-12-10T01:17:12,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/5b7b14e6e2cd4360905a32cfe56662b3 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3 2024-12-10T01:17:12,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3, entries=150, sequenceid=207, filesize=11.9 K 2024-12-10T01:17:12,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/deb95d21bba343d99b6fc54e34d8ebb6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6 2024-12-10T01:17:12,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6, entries=150, sequenceid=207, filesize=11.9 K 2024-12-10T01:17:12,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bff4d84e434d4670f70bf8a0a3136faf in 1256ms, sequenceid=207, compaction requested=true 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:12,718 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:12,718 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:12,719 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:12,719 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:12,719 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=108.4 K 2024-12-10T01:17:12,719 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/2d6978ed799240f3add6961de324240f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=36.0 K 2024-12-10T01:17:12,719 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066] 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6978ed799240f3add6961de324240f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab1c340dd8d548efa1c8c59f32dba6f2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting e236c89a4dd84dda8789bc09f640928b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733793428138 2024-12-10T01:17:12,719 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bace80c0e7654cd09951430f9779482d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733793428138 2024-12-10T01:17:12,720 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7b14e6e2cd4360905a32cfe56662b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:12,720 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff7f0330dc34033ab9ce08aaa087066, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:12,723 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:12,725 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:12,725 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412101852198aa37c453bbbb19264dd9824ae_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:12,725 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/a05dfdfd1c3949038361d78d8375b016 is 50, key is test_row_0/B:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:12,726 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412101852198aa37c453bbbb19264dd9824ae_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:12,726 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101852198aa37c453bbbb19264dd9824ae_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:12,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742353_1529 (size=12663) 2024-12-10T01:17:12,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742354_1530 (size=4469) 2024-12-10T01:17:12,730 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#445 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:12,730 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6e62a4dfaa747eb970b319d324e35b0 is 175, key is test_row_0/A:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:12,732 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/a05dfdfd1c3949038361d78d8375b016 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/a05dfdfd1c3949038361d78d8375b016 2024-12-10T01:17:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742355_1531 (size=31617) 2024-12-10T01:17:12,735 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into a05dfdfd1c3949038361d78d8375b016(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:12,735 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:12,735 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793432718; duration=0sec 2024-12-10T01:17:12,735 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:12,735 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:12,735 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:12,736 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:12,736 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:12,736 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:12,736 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b6c8c1e02a38417885db711abe4566c5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=36.0 K 2024-12-10T01:17:12,737 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/d6e62a4dfaa747eb970b319d324e35b0 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0 2024-12-10T01:17:12,737 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c8c1e02a38417885db711abe4566c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733793426949 2024-12-10T01:17:12,737 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting b5288246a5814bf5ae70ccbe28df6769, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733793428138 2024-12-10T01:17:12,737 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting deb95d21bba343d99b6fc54e34d8ebb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:12,739 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into d6e62a4dfaa747eb970b319d324e35b0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:12,739 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:12,739 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793432718; duration=0sec 2024-12-10T01:17:12,739 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:12,739 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:12,742 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#446 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:12,743 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/6f0acf3fe7584c64aa46bb6768eec568 is 50, key is test_row_0/C:col10/1733793431462/Put/seqid=0 2024-12-10T01:17:12,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742356_1532 (size=12663) 2024-12-10T01:17:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T01:17:12,799 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:12,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-12-10T01:17:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:12,801 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:12,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:12,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:12,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-10T01:17:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:12,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:12,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e0da5b45fced4cf393052b5df629092d_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793431480/Put/seqid=0 2024-12-10T01:17:12,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742357_1533 (size=12304) 2024-12-10T01:17:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:13,152 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/6f0acf3fe7584c64aa46bb6768eec568 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/6f0acf3fe7584c64aa46bb6768eec568 2024-12-10T01:17:13,156 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into 6f0acf3fe7584c64aa46bb6768eec568(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:13,156 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:13,156 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793432718; duration=0sec 2024-12-10T01:17:13,156 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:13,156 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:13,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e0da5b45fced4cf393052b5df629092d_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e0da5b45fced4cf393052b5df629092d_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/cca4fad0711842029051a0f426d4b46c, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:13,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/cca4fad0711842029051a0f426d4b46c is 175, key is test_row_0/A:col10/1733793431480/Put/seqid=0 2024-12-10T01:17:13,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742358_1534 (size=31105) 2024-12-10T01:17:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:13,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793493644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793493647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793493647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793493749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793493759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793493760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,769 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/cca4fad0711842029051a0f426d4b46c 2024-12-10T01:17:13,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/807b446943154c9e84ab1a49ecce7462 is 50, key is test_row_0/B:col10/1733793431480/Put/seqid=0 2024-12-10T01:17:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742359_1535 (size=12151) 2024-12-10T01:17:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793493959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793493969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793493970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,177 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/807b446943154c9e84ab1a49ecce7462 2024-12-10T01:17:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/5901429ed55049fb83cdf189f1828d45 is 50, key is test_row_0/C:col10/1733793431480/Put/seqid=0 2024-12-10T01:17:14,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742360_1536 (size=12151) 2024-12-10T01:17:14,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793494270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793494279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793494279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,586 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/5901429ed55049fb83cdf189f1828d45 2024-12-10T01:17:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/cca4fad0711842029051a0f426d4b46c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c 2024-12-10T01:17:14,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c, entries=150, sequenceid=226, filesize=30.4 K 2024-12-10T01:17:14,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/807b446943154c9e84ab1a49ecce7462 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462 2024-12-10T01:17:14,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462, entries=150, sequenceid=226, filesize=11.9 K 2024-12-10T01:17:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/5901429ed55049fb83cdf189f1828d45 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45 2024-12-10T01:17:14,598 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45, entries=150, sequenceid=226, filesize=11.9 K 2024-12-10T01:17:14,599 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for bff4d84e434d4670f70bf8a0a3136faf in 1646ms, sequenceid=226, compaction requested=false 2024-12-10T01:17:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-12-10T01:17:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-12-10T01:17:14,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-10T01:17:14,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-12-10T01:17:14,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.8020 sec 2024-12-10T01:17:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:14,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:14,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b5e2b30e9b804791b2f418a08792c288_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742361_1537 (size=14794) 2024-12-10T01:17:14,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793494800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793494801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793494802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793494909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793494909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793494909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T01:17:14,949 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:14,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T01:17:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-10T01:17:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:14,951 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T01:17:14,951 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T01:17:14,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T01:17:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:15,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793495119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793495119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793495120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,196 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:15,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b5e2b30e9b804791b2f418a08792c288_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5e2b30e9b804791b2f418a08792c288_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:15,199 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/56188cf71cac42b581fa8a7ff7bf0cc9, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:15,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/56188cf71cac42b581fa8a7ff7bf0cc9 is 175, key is test_row_0/A:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:15,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742362_1538 (size=39749) 2024-12-10T01:17:15,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:15,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793495429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793495429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793495429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:15,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/56188cf71cac42b581fa8a7ff7bf0cc9 2024-12-10T01:17:15,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d31f8fd610de44bb82ab2a8a48cf8608 is 50, key is test_row_0/B:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742363_1539 (size=12151) 2024-12-10T01:17:15,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:15,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:15,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1733793495939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1733793495939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44878 deadline: 1733793495939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:16,005 INFO [Thread-2135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2135. 2024-12-10T01:17:16,005 DEBUG [Thread-2135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,005 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,005 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,006 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-12-10T01:17:16,006 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,006 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,006 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,007 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-12-10T01:17:16,007 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,007 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,007 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,008 INFO [Thread-2129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2129. 2024-12-10T01:17:16,008 DEBUG [Thread-2129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,008 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,008 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,009 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-12-10T01:17:16,009 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,009 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,009 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:16,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:16,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d31f8fd610de44bb82ab2a8a48cf8608 2024-12-10T01:17:16,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c1787d98ad774847aa5f30a0e8a80796 is 50, key is test_row_0/C:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:16,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742364_1540 (size=12151) 2024-12-10T01:17:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:16,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:16,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:16,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T01:17:16,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c1787d98ad774847aa5f30a0e8a80796 2024-12-10T01:17:16,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T01:17:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44890 deadline: 1733793496460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:16,461 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd1a2b66caf8,38925,1733793262612 for 'test_row_0' in bff4d84e434d4670f70bf8a0a3136faf of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T01:17:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 , the old value is region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/56188cf71cac42b581fa8a7ff7bf0cc9 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9 2024-12-10T01:17:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T01:17:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., hostname=fd1a2b66caf8,38925,1733793262612, seqNum=5 because the exception is null or not the one we care about 2024-12-10T01:17:16,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9, entries=200, sequenceid=243, filesize=38.8 K 2024-12-10T01:17:16,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/d31f8fd610de44bb82ab2a8a48cf8608 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608 2024-12-10T01:17:16,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608, entries=150, sequenceid=243, filesize=11.9 K 2024-12-10T01:17:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/c1787d98ad774847aa5f30a0e8a80796 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796 2024-12-10T01:17:16,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796, entries=150, sequenceid=243, filesize=11.9 K 2024-12-10T01:17:16,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for bff4d84e434d4670f70bf8a0a3136faf in 1694ms, sequenceid=243, compaction requested=true 2024-12-10T01:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:16,477 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bff4d84e434d4670f70bf8a0a3136faf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T01:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:16,477 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/B is initiating minor compaction (all files) 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/A is initiating minor compaction (all files) 2024-12-10T01:17:16,478 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/A in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,478 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/B in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,478 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=100.1 K 2024-12-10T01:17:16,478 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/a05dfdfd1c3949038361d78d8375b016, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=36.1 K 2024-12-10T01:17:16,478 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. files: [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9] 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting a05dfdfd1c3949038361d78d8375b016, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e62a4dfaa747eb970b319d324e35b0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:16,478 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 807b446943154c9e84ab1a49ecce7462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733793431477 2024-12-10T01:17:16,479 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca4fad0711842029051a0f426d4b46c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733793431477 2024-12-10T01:17:16,479 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting d31f8fd610de44bb82ab2a8a48cf8608, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733793433641 2024-12-10T01:17:16,479 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56188cf71cac42b581fa8a7ff7bf0cc9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733793433641 2024-12-10T01:17:16,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-10T01:17:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:16,484 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:16,485 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:16,485 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/63e01a5aa5bc4590849fbf923e2e9d0d is 50, key is test_row_0/B:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:16,486 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104be37252b024414ea6c3b0171f949608_bff4d84e434d4670f70bf8a0a3136faf store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:16,488 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104be37252b024414ea6c3b0171f949608_bff4d84e434d4670f70bf8a0a3136faf, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:16,489 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104be37252b024414ea6c3b0171f949608_bff4d84e434d4670f70bf8a0a3136faf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ae82789442c040d1b3eff5b086253716_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793434795/Put/seqid=0 2024-12-10T01:17:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742365_1541 (size=12765) 2024-12-10T01:17:16,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742366_1542 (size=4469) 2024-12-10T01:17:16,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742367_1543 (size=12404) 2024-12-10T01:17:16,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:16,897 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:16,898 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/a583c9be142446f5a235f4cabac657a8 is 175, key is test_row_0/A:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:16,901 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/63e01a5aa5bc4590849fbf923e2e9d0d as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/63e01a5aa5bc4590849fbf923e2e9d0d 2024-12-10T01:17:16,902 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ae82789442c040d1b3eff5b086253716_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ae82789442c040d1b3eff5b086253716_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:16,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742368_1544 (size=31719) 2024-12-10T01:17:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/65fef3bafb7b4f01aba348816b573eae, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/65fef3bafb7b4f01aba348816b573eae is 175, key is test_row_0/A:col10/1733793434795/Put/seqid=0 2024-12-10T01:17:16,906 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/B of bff4d84e434d4670f70bf8a0a3136faf into 63e01a5aa5bc4590849fbf923e2e9d0d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:16,906 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:16,906 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/B, priority=13, startTime=1733793436477; duration=0sec 2024-12-10T01:17:16,906 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T01:17:16,906 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:B 2024-12-10T01:17:16,906 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T01:17:16,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742369_1545 (size=31205) 2024-12-10T01:17:16,907 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T01:17:16,907 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1541): bff4d84e434d4670f70bf8a0a3136faf/C is initiating minor compaction (all files) 2024-12-10T01:17:16,907 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bff4d84e434d4670f70bf8a0a3136faf/C in TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:16,907 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/6f0acf3fe7584c64aa46bb6768eec568, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp, totalSize=36.1 K 2024-12-10T01:17:16,907 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f0acf3fe7584c64aa46bb6768eec568, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733793430317 2024-12-10T01:17:16,907 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5901429ed55049fb83cdf189f1828d45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733793431477 2024-12-10T01:17:16,908 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] compactions.Compactor(225): Compacting c1787d98ad774847aa5f30a0e8a80796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733793433641 2024-12-10T01:17:16,912 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bff4d84e434d4670f70bf8a0a3136faf#C#compaction#456 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T01:17:16,913 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/f8927800ff6841b7bac81536697f511b is 50, key is test_row_0/C:col10/1733793433646/Put/seqid=0 2024-12-10T01:17:16,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742370_1546 (size=12765) 2024-12-10T01:17:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38925 {}] regionserver.HRegion(8855): Flush requested on bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:16,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. as already flushing 2024-12-10T01:17:16,951 INFO [Thread-2118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2118. 2024-12-10T01:17:16,952 DEBUG [Thread-2118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,952 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,952 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,962 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-12-10T01:17:16,962 INFO [Thread-2124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2124. 2024-12-10T01:17:16,962 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,962 DEBUG [Thread-2124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:16,962 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,962 DEBUG [Thread-2124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,962 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,962 DEBUG [Thread-2124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:16,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:16,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:17,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/65fef3bafb7b4f01aba348816b573eae 2024-12-10T01:17:17,314 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/a583c9be142446f5a235f4cabac657a8 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/a583c9be142446f5a235f4cabac657a8 2024-12-10T01:17:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/cd12256002cc4ee691ccb0727519d298 is 50, key is test_row_0/B:col10/1733793434795/Put/seqid=0 2024-12-10T01:17:17,319 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/A of bff4d84e434d4670f70bf8a0a3136faf into a583c9be142446f5a235f4cabac657a8(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:17,319 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:17,320 INFO [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/A, priority=13, startTime=1733793436477; duration=0sec 2024-12-10T01:17:17,320 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:17,320 DEBUG [RS:0;fd1a2b66caf8:38925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:A 2024-12-10T01:17:17,320 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/f8927800ff6841b7bac81536697f511b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f8927800ff6841b7bac81536697f511b 2024-12-10T01:17:17,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742371_1547 (size=12251) 2024-12-10T01:17:17,325 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bff4d84e434d4670f70bf8a0a3136faf/C of bff4d84e434d4670f70bf8a0a3136faf into f8927800ff6841b7bac81536697f511b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T01:17:17,325 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:17,325 INFO [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., storeName=bff4d84e434d4670f70bf8a0a3136faf/C, priority=13, startTime=1733793436477; duration=0sec 2024-12-10T01:17:17,325 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T01:17:17,326 DEBUG [RS:0;fd1a2b66caf8:38925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bff4d84e434d4670f70bf8a0a3136faf:C 2024-12-10T01:17:17,724 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/cd12256002cc4ee691ccb0727519d298 2024-12-10T01:17:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af387cea104241ee8237a23dc81acf86 is 50, key is test_row_0/C:col10/1733793434795/Put/seqid=0 2024-12-10T01:17:17,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742372_1548 (size=12251) 2024-12-10T01:17:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af387cea104241ee8237a23dc81acf86 2024-12-10T01:17:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/65fef3bafb7b4f01aba348816b573eae as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/65fef3bafb7b4f01aba348816b573eae 2024-12-10T01:17:18,157 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/65fef3bafb7b4f01aba348816b573eae, entries=150, sequenceid=259, filesize=30.5 K 2024-12-10T01:17:18,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/cd12256002cc4ee691ccb0727519d298 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/cd12256002cc4ee691ccb0727519d298 2024-12-10T01:17:18,159 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/cd12256002cc4ee691ccb0727519d298, entries=150, sequenceid=259, filesize=12.0 K 2024-12-10T01:17:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/af387cea104241ee8237a23dc81acf86 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af387cea104241ee8237a23dc81acf86 2024-12-10T01:17:18,162 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af387cea104241ee8237a23dc81acf86, entries=150, sequenceid=259, filesize=12.0 K 2024-12-10T01:17:18,163 INFO [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for bff4d84e434d4670f70bf8a0a3136faf in 1682ms, sequenceid=259, compaction requested=false 2024-12-10T01:17:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for bff4d84e434d4670f70bf8a0a3136faf: 2024-12-10T01:17:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd1a2b66caf8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-10T01:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-10T01:17:18,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-10T01:17:18,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2130 sec 2024-12-10T01:17:18,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 3.2170 sec 2024-12-10T01:17:18,545 INFO [Thread-2122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2122. 2024-12-10T01:17:18,545 DEBUG [Thread-2122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:18,545 DEBUG [Thread-2122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:18,545 DEBUG [Thread-2122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:18,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T01:17:19,131 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:21,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T01:17:26,544 INFO [Thread-2120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2120. 2024-12-10T01:17:26,544 DEBUG [Thread-2120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T01:17:26,544 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:26,544 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:26,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5170 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4963 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5026 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5185 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4960 2024-12-10T01:17:26,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T01:17:26,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:17:26,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:17:26,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:26,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:26,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:26,548 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T01:17:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:26,553 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793446553"}]},"ts":"1733793446553"} 2024-12-10T01:17:26,555 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T01:17:26,555 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T01:17:26,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T01:17:26,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, UNASSIGN}] 2024-12-10T01:17:26,559 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, UNASSIGN 2024-12-10T01:17:26,560 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=146 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=CLOSING, regionLocation=fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:26,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, UNASSIGN because future has completed 2024-12-10T01:17:26,562 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T01:17:26,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612}] 2024-12-10T01:17:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:26,715 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(122): Close bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1722): Closing bff4d84e434d4670f70bf8a0a3136faf, disabling compactions & flushes 2024-12-10T01:17:26,715 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. after waiting 0 ms 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:26,715 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(2902): Flushing bff4d84e434d4670f70bf8a0a3136faf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=A 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=B 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bff4d84e434d4670f70bf8a0a3136faf, store=C 2024-12-10T01:17:26,715 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T01:17:26,722 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b1261b490f2b491bb68581e85bf652da_bff4d84e434d4670f70bf8a0a3136faf is 50, key is test_row_0/A:col10/1733793446541/Put/seqid=0 2024-12-10T01:17:26,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742373_1549 (size=9914) 2024-12-10T01:17:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:27,128 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T01:17:27,138 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b1261b490f2b491bb68581e85bf652da_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1261b490f2b491bb68581e85bf652da_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:27,139 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8ff949ce87534ecb83bc72760a61dbf1, store: [table=TestAcidGuarantees family=A region=bff4d84e434d4670f70bf8a0a3136faf] 2024-12-10T01:17:27,140 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8ff949ce87534ecb83bc72760a61dbf1 is 175, key is test_row_0/A:col10/1733793446541/Put/seqid=0 2024-12-10T01:17:27,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742374_1550 (size=22561) 2024-12-10T01:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:27,546 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8ff949ce87534ecb83bc72760a61dbf1 2024-12-10T01:17:27,558 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/1c72cc2447ad44a08e1777f9cdaa941a is 50, key is test_row_0/B:col10/1733793446541/Put/seqid=0 2024-12-10T01:17:27,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742375_1551 (size=9857) 2024-12-10T01:17:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:27,964 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/1c72cc2447ad44a08e1777f9cdaa941a 2024-12-10T01:17:27,977 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/418ce462cc9c42c0b02a6370519038bf is 50, key is test_row_0/C:col10/1733793446541/Put/seqid=0 2024-12-10T01:17:27,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742376_1552 (size=9857) 2024-12-10T01:17:28,383 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/418ce462cc9c42c0b02a6370519038bf 2024-12-10T01:17:28,393 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/A/8ff949ce87534ecb83bc72760a61dbf1 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8ff949ce87534ecb83bc72760a61dbf1 2024-12-10T01:17:28,396 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8ff949ce87534ecb83bc72760a61dbf1, entries=100, sequenceid=270, filesize=22.0 K 2024-12-10T01:17:28,396 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/B/1c72cc2447ad44a08e1777f9cdaa941a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/1c72cc2447ad44a08e1777f9cdaa941a 2024-12-10T01:17:28,399 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/1c72cc2447ad44a08e1777f9cdaa941a, entries=100, sequenceid=270, filesize=9.6 K 2024-12-10T01:17:28,400 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/.tmp/C/418ce462cc9c42c0b02a6370519038bf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/418ce462cc9c42c0b02a6370519038bf 2024-12-10T01:17:28,403 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/418ce462cc9c42c0b02a6370519038bf, entries=100, sequenceid=270, filesize=9.6 K 2024-12-10T01:17:28,403 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bff4d84e434d4670f70bf8a0a3136faf in 1688ms, sequenceid=270, compaction requested=true 2024-12-10T01:17:28,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9] to archive 2024-12-10T01:17:28,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:17:28,407 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8c40648293d747d2b1ecf942ce5d7858 2024-12-10T01:17:28,407 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b1596bf46da5423bbb0ad1386aeb2bc1 2024-12-10T01:17:28,407 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6cf3fcfb69948dea036e5ca9433403a 2024-12-10T01:17:28,407 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/3da34426c7e04940a851a2eb09aebe6a 2024-12-10T01:17:28,407 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/85311e4662ee4756b9d2f76eaa1980aa 2024-12-10T01:17:28,408 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/fbf6194a711342809de42d45ada46e11 2024-12-10T01:17:28,408 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bc7cd21158b64f4b8d9e095bc1a63831 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0700ed90d58848d8acfa9354517785a5 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/205ed856ea3547c29ffb29d1497e9438 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ff5624d6e4084d9ca730c3a82e200bab 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/e139d3dda0d947fdbfb6d48ebc7702d8 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/b3220c5843284b15b5a0d469c307435b 2024-12-10T01:17:28,409 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/ab1c340dd8d548efa1c8c59f32dba6f2 2024-12-10T01:17:28,410 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/bace80c0e7654cd09951430f9779482d 2024-12-10T01:17:28,410 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/0ff7f0330dc34033ab9ce08aaa087066 2024-12-10T01:17:28,410 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/cca4fad0711842029051a0f426d4b46c 2024-12-10T01:17:28,410 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/d6e62a4dfaa747eb970b319d324e35b0 2024-12-10T01:17:28,411 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/56188cf71cac42b581fa8a7ff7bf0cc9 2024-12-10T01:17:28,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/ec5ad2cbefde4967b19c79f7fe53e5ba, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/f5f330cbc80a4319a65c96f8b8c912d5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/53e92106ca2a4bc29e0d4979f4d5c542, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/2d6978ed799240f3add6961de324240f, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/a05dfdfd1c3949038361d78d8375b016, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608] to archive 2024-12-10T01:17:28,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/b139e94efdbe44f480ef91dae731265d 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/ec5ad2cbefde4967b19c79f7fe53e5ba to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/ec5ad2cbefde4967b19c79f7fe53e5ba 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/96b8ef0a062342a4a0b12db669445b96 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/f5f330cbc80a4319a65c96f8b8c912d5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/f5f330cbc80a4319a65c96f8b8c912d5 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d1ad89e808fb4c31bf41cda1bd5aac08 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/3c1e99dbc5804b60a2a7730b61296cb5 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c3334cec09ca4909a829ce252c722b89 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/941c5722c0f14883a87368506781c06e 2024-12-10T01:17:28,415 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/53e92106ca2a4bc29e0d4979f4d5c542 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/53e92106ca2a4bc29e0d4979f4d5c542 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/c2f264e8d80b4f5a99123563b3f408cb 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/15a6ae9139bb476e92051ebb189d190b 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/fc7f1a65549144d08f12feea6ffe9e8a 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/2d6978ed799240f3add6961de324240f to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/2d6978ed799240f3add6961de324240f 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/e236c89a4dd84dda8789bc09f640928b 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/a05dfdfd1c3949038361d78d8375b016 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/a05dfdfd1c3949038361d78d8375b016 2024-12-10T01:17:28,416 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/5b7b14e6e2cd4360905a32cfe56662b3 2024-12-10T01:17:28,417 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/807b446943154c9e84ab1a49ecce7462 2024-12-10T01:17:28,417 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/d31f8fd610de44bb82ab2a8a48cf8608 2024-12-10T01:17:28,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d63640b5e6334f8dbb675a4e0ee63c76, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/2dcb18d5f7cd40bb919ed0a01274a0f1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f70c3d14b10748c0a1271a7764041750, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b6c8c1e02a38417885db711abe4566c5, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/6f0acf3fe7584c64aa46bb6768eec568, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796] to archive 2024-12-10T01:17:28,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d3d602c73bfd4dd3ad73584d5151fb17 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/089222c3780942aeb2e793648cc16783 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d63640b5e6334f8dbb675a4e0ee63c76 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/d63640b5e6334f8dbb675a4e0ee63c76 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af9e58599219401d9576ac9b6696d64c 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/2dcb18d5f7cd40bb919ed0a01274a0f1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/2dcb18d5f7cd40bb919ed0a01274a0f1 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/33e1706f29994b4d83672e3cf7a99dc6 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/8181c4c73b634cd798426f0a28153bcf 2024-12-10T01:17:28,421 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/970198c69131421e8ca81a240492e1a9 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f70c3d14b10748c0a1271a7764041750 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f70c3d14b10748c0a1271a7764041750 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c48c3e7df6324519b3963840816cb0e1 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/82f97a712800415bb9e589d2f2c91b35 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b6c8c1e02a38417885db711abe4566c5 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b6c8c1e02a38417885db711abe4566c5 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/b5288246a5814bf5ae70ccbe28df6769 2024-12-10T01:17:28,422 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/6f0acf3fe7584c64aa46bb6768eec568 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/6f0acf3fe7584c64aa46bb6768eec568 2024-12-10T01:17:28,423 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/e347885ace8c4ea3a15205308d505ac1 2024-12-10T01:17:28,423 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/deb95d21bba343d99b6fc54e34d8ebb6 2024-12-10T01:17:28,423 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/5901429ed55049fb83cdf189f1828d45 2024-12-10T01:17:28,423 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/c1787d98ad774847aa5f30a0e8a80796 2024-12-10T01:17:28,426 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits/273.seqid, newMaxSeqId=273, maxSeqId=4 2024-12-10T01:17:28,427 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. 2024-12-10T01:17:28,427 DEBUG [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1676): Region close journal for bff4d84e434d4670f70bf8a0a3136faf: Waiting for close lock at 1733793446715Running coprocessor pre-close hooks at 1733793446715Disabling compacts and flushes for region at 1733793446715Disabling writes for close at 1733793446715Obtaining lock to block concurrent updates at 1733793446715Preparing flush snapshotting stores in bff4d84e434d4670f70bf8a0a3136faf at 1733793446715Finished memstore snapshotting TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733793446715Flushing stores of TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf. at 1733793446716 (+1 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/A: creating writer at 1733793446716Flushing mob file bff4d84e434d4670f70bf8a0a3136faf/A: appending metadata at 1733793446721 (+5 ms)Flushing mob file bff4d84e434d4670f70bf8a0a3136faf/A: closing flushed file at 1733793446721Flushing bff4d84e434d4670f70bf8a0a3136faf/A: appending metadata at 1733793447140 (+419 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/A: closing flushed file at 1733793447140Flushing bff4d84e434d4670f70bf8a0a3136faf/B: creating writer at 1733793447554 (+414 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/B: appending metadata at 1733793447557 (+3 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/B: closing flushed file at 1733793447557Flushing bff4d84e434d4670f70bf8a0a3136faf/C: creating writer at 1733793447972 (+415 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/C: appending metadata at 1733793447976 (+4 ms)Flushing bff4d84e434d4670f70bf8a0a3136faf/C: closing flushed file at 1733793447976Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba51317: reopening flushed file at 1733793448392 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22742817: reopening flushed file at 1733793448396 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@273746fb: reopening flushed file at 1733793448399 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bff4d84e434d4670f70bf8a0a3136faf in 1688ms, sequenceid=270, compaction requested=true at 1733793448403 (+4 ms)Writing region close event to WAL at 1733793448424 (+21 ms)Running coprocessor post-close hooks at 1733793448427 (+3 ms)Closed at 1733793448427 2024-12-10T01:17:28,428 INFO [RS_CLOSE_REGION-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(157): Closed bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=146 updating hbase:meta row=bff4d84e434d4670f70bf8a0a3136faf, regionState=CLOSED 2024-12-10T01:17:28,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 because future has completed 2024-12-10T01:17:28,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-10T01:17:28,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; CloseRegionProcedure bff4d84e434d4670f70bf8a0a3136faf, server=fd1a2b66caf8,38925,1733793262612 in 1.8680 sec 2024-12-10T01:17:28,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-10T01:17:28,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bff4d84e434d4670f70bf8a0a3136faf, UNASSIGN in 1.8730 sec 2024-12-10T01:17:28,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-10T01:17:28,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-12-10T01:17:28,436 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733793448436"}]},"ts":"1733793448436"} 2024-12-10T01:17:28,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T01:17:28,437 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T01:17:28,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8900 sec 2024-12-10T01:17:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T01:17:28,711 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:28,713 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T01:17:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,717 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T01:17:28,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,721 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,724 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C, FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits] 2024-12-10T01:17:28,727 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8ff949ce87534ecb83bc72760a61dbf1 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/8ff949ce87534ecb83bc72760a61dbf1 2024-12-10T01:17:28,727 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/65fef3bafb7b4f01aba348816b573eae to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/65fef3bafb7b4f01aba348816b573eae 2024-12-10T01:17:28,727 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/a583c9be142446f5a235f4cabac657a8 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/A/a583c9be142446f5a235f4cabac657a8 2024-12-10T01:17:28,729 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/1c72cc2447ad44a08e1777f9cdaa941a to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/1c72cc2447ad44a08e1777f9cdaa941a 2024-12-10T01:17:28,729 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/cd12256002cc4ee691ccb0727519d298 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/cd12256002cc4ee691ccb0727519d298 2024-12-10T01:17:28,730 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/63e01a5aa5bc4590849fbf923e2e9d0d to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/B/63e01a5aa5bc4590849fbf923e2e9d0d 2024-12-10T01:17:28,731 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/418ce462cc9c42c0b02a6370519038bf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/418ce462cc9c42c0b02a6370519038bf 2024-12-10T01:17:28,732 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f8927800ff6841b7bac81536697f511b to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/f8927800ff6841b7bac81536697f511b 2024-12-10T01:17:28,732 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af387cea104241ee8237a23dc81acf86 to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/C/af387cea104241ee8237a23dc81acf86 2024-12-10T01:17:28,734 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits/273.seqid to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf/recovered.edits/273.seqid 2024-12-10T01:17:28,734 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/default/TestAcidGuarantees/bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,734 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T01:17:28,734 DEBUG [PEWorker-5 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:17:28,735 DEBUG [PEWorker-5 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T01:17:28,739 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121008d3033ece294b46924edada838f1832_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121008d3033ece294b46924edada838f1832_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,739 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101e5c49f427b34d1bb62ab36af7647fbb_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101e5c49f427b34d1bb62ab36af7647fbb_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210085bad8de9ff402eba1a1fb22ad7aa04_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210085bad8de9ff402eba1a1fb22ad7aa04_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121021cd5ae268a54e66a46ac2cd663128c5_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121021cd5ae268a54e66a46ac2cd663128c5_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121066ae2517d3c94fa4a5d9fb8c353fcf47_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121066ae2517d3c94fa4a5d9fb8c353fcf47_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107af77f65e5a34de799f2a19cb804b73e_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107af77f65e5a34de799f2a19cb804b73e_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106fa3271b9b504b1c99e6e51ecd5b88de_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106fa3271b9b504b1c99e6e51ecd5b88de_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,740 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ae82789442c040d1b3eff5b086253716_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ae82789442c040d1b3eff5b086253716_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,741 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1261b490f2b491bb68581e85bf652da_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1261b490f2b491bb68581e85bf652da_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,741 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5e2b30e9b804791b2f418a08792c288_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b5e2b30e9b804791b2f418a08792c288_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,741 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d560048c7fa94840af2517897d021eb2_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d560048c7fa94840af2517897d021eb2_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,742 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e0da5b45fced4cf393052b5df629092d_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e0da5b45fced4cf393052b5df629092d_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,742 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3b9a018fa434aa0868cd3e0f4955a33_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3b9a018fa434aa0868cd3e0f4955a33_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,742 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d08321697b614e5d8a6f6f1ec70d47e6_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d08321697b614e5d8a6f6f1ec70d47e6_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,742 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d1b008c9ddb84a63829ca50a5c4031de_bff4d84e434d4670f70bf8a0a3136faf to hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d1b008c9ddb84a63829ca50a5c4031de_bff4d84e434d4670f70bf8a0a3136faf 2024-12-10T01:17:28,742 DEBUG [PEWorker-5 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T01:17:28,744 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,746 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T01:17:28,748 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T01:17:28,749 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,749 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T01:17:28,749 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733793448749"}]},"ts":"9223372036854775807"} 2024-12-10T01:17:28,750 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T01:17:28,750 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bff4d84e434d4670f70bf8a0a3136faf, NAME => 'TestAcidGuarantees,,1733793412851.bff4d84e434d4670f70bf8a0a3136faf.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T01:17:28,750 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T01:17:28,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733793448750"}]},"ts":"9223372036854775807"} 2024-12-10T01:17:28,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T01:17:28,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T01:17:28,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-10T01:17:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37249 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T01:17:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T01:17:28,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T01:17:28,844 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254 (was 255), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 337), ProcessCount=11 (was 11), AvailableMemoryMB=7316 (was 7351) 2024-12-10T01:17:28,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T01:17:28,844 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:17:28,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:17:28,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:28,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:28,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T01:17:28,845 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T01:17:28,845 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-12-10T01:17:28,845 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fd1a2b66caf8,37249,1733793261894 2024-12-10T01:17:28,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T01:17:28,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T01:17:28,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:17:28,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:17:28,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T01:17:28,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T01:17:28,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T01:17:28,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:17:28,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:28,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T01:17:28,905 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fd1a2b66caf8,38925,1733793262612' ***** 2024-12-10T01:17:28,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T01:17:28,906 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T01:17:28,906 INFO [RS:0;fd1a2b66caf8:38925 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T01:17:28,906 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T01:17:28,906 INFO [RS:0;fd1a2b66caf8:38925 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(959): stopping server fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fd1a2b66caf8:38925. 2024-12-10T01:17:28,907 DEBUG [RS:0;fd1a2b66caf8:38925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T01:17:28,907 DEBUG [RS:0;fd1a2b66caf8:38925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T01:17:28,907 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T01:17:28,908 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T01:17:28,908 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T01:17:28,908 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:28,908 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T01:17:28,908 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T01:17:28,908 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T01:17:28,908 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T01:17:28,908 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T01:17:28,908 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-10T01:17:28,928 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/info/95d7e3f44b8942f78e8b0ae8f1887766 is 95, key is TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af./info:/1733793295465/DeleteFamily/seqid=0 2024-12-10T01:17:28,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742377_1553 (size=6514) 2024-12-10T01:17:28,964 INFO [regionserver/fd1a2b66caf8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T01:17:29,108 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:29,309 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:29,333 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/info/95d7e3f44b8942f78e8b0ae8f1887766 2024-12-10T01:17:29,361 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/ns/008114b495af47059090e197834ad0bf is 93, key is TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af./ns:/1733793295412/DeleteFamily/seqid=0 2024-12-10T01:17:29,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742378_1554 (size=5906) 2024-12-10T01:17:29,509 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:29,565 INFO [regionserver/fd1a2b66caf8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T01:17:29,565 INFO [regionserver/fd1a2b66caf8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T01:17:29,709 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:29,765 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/ns/008114b495af47059090e197834ad0bf 2024-12-10T01:17:29,781 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/rep_barrier/f27a919bca71411baf03cc1c6d9af52a is 102, key is TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af./rep_barrier:/1733793295412/DeleteFamily/seqid=0 2024-12-10T01:17:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742379_1555 (size=6025) 2024-12-10T01:17:29,910 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:30,110 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:30,186 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/rep_barrier/f27a919bca71411baf03cc1c6d9af52a 2024-12-10T01:17:30,213 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/table/1e85d008cf67498b8a56260bed36928c is 96, key is TestAcidGuarantees,,1733793267309.5c85b06919b802618bd08e23219e27af./table:/1733793295412/DeleteFamily/seqid=0 2024-12-10T01:17:30,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742380_1556 (size=5936) 2024-12-10T01:17:30,310 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:30,511 DEBUG [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T01:17:30,618 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/table/1e85d008cf67498b8a56260bed36928c 2024-12-10T01:17:30,627 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/info/95d7e3f44b8942f78e8b0ae8f1887766 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/info/95d7e3f44b8942f78e8b0ae8f1887766 2024-12-10T01:17:30,632 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/info/95d7e3f44b8942f78e8b0ae8f1887766, entries=12, sequenceid=90, filesize=6.4 K 2024-12-10T01:17:30,634 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/ns/008114b495af47059090e197834ad0bf as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/ns/008114b495af47059090e197834ad0bf 2024-12-10T01:17:30,637 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/ns/008114b495af47059090e197834ad0bf, entries=8, sequenceid=90, filesize=5.8 K 2024-12-10T01:17:30,638 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/rep_barrier/f27a919bca71411baf03cc1c6d9af52a as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/rep_barrier/f27a919bca71411baf03cc1c6d9af52a 2024-12-10T01:17:30,641 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/rep_barrier/f27a919bca71411baf03cc1c6d9af52a, entries=6, sequenceid=90, filesize=5.9 K 2024-12-10T01:17:30,642 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/.tmp/table/1e85d008cf67498b8a56260bed36928c as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/table/1e85d008cf67498b8a56260bed36928c 2024-12-10T01:17:30,645 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/table/1e85d008cf67498b8a56260bed36928c, entries=7, sequenceid=90, filesize=5.8 K 2024-12-10T01:17:30,646 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1738ms, sequenceid=90, compaction requested=false 2024-12-10T01:17:30,651 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-10T01:17:30,652 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T01:17:30,652 INFO [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T01:17:30,652 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733793448908Running coprocessor pre-close hooks at 1733793448908Disabling compacts and flushes for region at 1733793448908Disabling writes for close at 1733793448908Obtaining lock to block concurrent updates at 1733793448908Preparing flush snapshotting stores in 1588230740 at 1733793448908Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733793448909 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733793448910 (+1 ms)Flushing 1588230740/info: creating writer at 1733793448910Flushing 1588230740/info: appending metadata at 1733793448928 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733793448928Flushing 1588230740/ns: creating writer at 1733793449343 (+415 ms)Flushing 1588230740/ns: appending metadata at 1733793449361 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733793449361Flushing 1588230740/rep_barrier: creating writer at 1733793449768 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733793449781 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733793449781Flushing 1588230740/table: creating writer at 1733793450196 (+415 ms)Flushing 1588230740/table: appending metadata at 1733793450213 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733793450213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e299e7c: reopening flushed file at 1733793450626 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3273a6: reopening flushed file at 1733793450633 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b231102: reopening flushed file at 1733793450637 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44971fd0: reopening flushed file at 1733793450641 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1738ms, sequenceid=90, compaction requested=false at 1733793450646 (+5 ms)Writing region close event to WAL at 1733793450648 (+2 ms)Running coprocessor post-close hooks at 1733793450652 (+4 ms)Closed at 1733793450652 2024-12-10T01:17:30,652 DEBUG [RS_CLOSE_META-regionserver/fd1a2b66caf8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T01:17:30,711 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(976): stopping server fd1a2b66caf8,38925,1733793262612; all regions closed. 2024-12-10T01:17:30,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741834_1010 (size=25344) 2024-12-10T01:17:30,723 DEBUG [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/oldWALs 2024-12-10T01:17:30,723 INFO [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fd1a2b66caf8%2C38925%2C1733793262612.meta:.meta(num 1733793266845) 2024-12-10T01:17:30,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741832_1008 (size=19603338) 2024-12-10T01:17:30,728 DEBUG [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/oldWALs 2024-12-10T01:17:30,728 INFO [RS:0;fd1a2b66caf8:38925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fd1a2b66caf8%2C38925%2C1733793262612:(num 1733793265715) 2024-12-10T01:17:30,728 DEBUG [RS:0;fd1a2b66caf8:38925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T01:17:30,728 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T01:17:30,728 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T01:17:30,728 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.ChoreService(370): Chore service for: regionserver/fd1a2b66caf8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T01:17:30,729 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T01:17:30,729 INFO [regionserver/fd1a2b66caf8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T01:17:30,730 INFO [RS:0;fd1a2b66caf8:38925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38925 2024-12-10T01:17:30,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T01:17:30,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fd1a2b66caf8,38925,1733793262612 2024-12-10T01:17:30,786 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T01:17:30,796 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fd1a2b66caf8,38925,1733793262612] 2024-12-10T01:17:30,806 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fd1a2b66caf8,38925,1733793262612 already deleted, retry=false 2024-12-10T01:17:30,806 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fd1a2b66caf8,38925,1733793262612 expired; onlineServers=0 2024-12-10T01:17:30,807 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fd1a2b66caf8,37249,1733793261894' ***** 2024-12-10T01:17:30,807 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T01:17:30,807 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T01:17:30,807 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T01:17:30,807 DEBUG [M:0;fd1a2b66caf8:37249 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T01:17:30,807 DEBUG [M:0;fd1a2b66caf8:37249 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T01:17:30,807 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T01:17:30,807 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.large.0-1733793265431 {}] cleaner.HFileCleaner(306): Exit Thread[master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.large.0-1733793265431,5,FailOnTimeoutGroup] 2024-12-10T01:17:30,807 DEBUG [master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.small.0-1733793265433 {}] cleaner.HFileCleaner(306): Exit Thread[master/fd1a2b66caf8:0:becomeActiveMaster-HFileCleaner.small.0-1733793265433,5,FailOnTimeoutGroup] 2024-12-10T01:17:30,807 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.ChoreService(370): Chore service for: master/fd1a2b66caf8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T01:17:30,807 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T01:17:30,808 DEBUG [M:0;fd1a2b66caf8:37249 {}] master.HMaster(1795): Stopping service threads 2024-12-10T01:17:30,808 INFO [M:0;fd1a2b66caf8:37249 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T01:17:30,808 INFO [M:0;fd1a2b66caf8:37249 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T01:17:30,808 INFO [M:0;fd1a2b66caf8:37249 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T01:17:30,809 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T01:17:30,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T01:17:30,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T01:17:30,817 DEBUG [M:0;fd1a2b66caf8:37249 {}] zookeeper.ZKUtil(347): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T01:17:30,817 WARN [M:0;fd1a2b66caf8:37249 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T01:17:30,818 INFO [M:0;fd1a2b66caf8:37249 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/.lastflushedseqids 2024-12-10T01:17:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742381_1557 (size=72) 2024-12-10T01:17:30,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T01:17:30,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38925-0x1000d49acf30001, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T01:17:30,897 INFO [RS:0;fd1a2b66caf8:38925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T01:17:30,898 INFO [RS:0;fd1a2b66caf8:38925 {}] regionserver.HRegionServer(1031): Exiting; stopping=fd1a2b66caf8,38925,1733793262612; zookeeper connection closed. 2024-12-10T01:17:30,898 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28e5bc67 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28e5bc67 2024-12-10T01:17:30,899 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T01:17:31,232 INFO [M:0;fd1a2b66caf8:37249 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T01:17:31,232 INFO [M:0;fd1a2b66caf8:37249 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T01:17:31,232 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T01:17:31,232 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:17:31,232 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:17:31,233 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T01:17:31,233 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:17:31,233 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=736.03 KB heapSize=899.45 KB 2024-12-10T01:17:31,255 DEBUG [M:0;fd1a2b66caf8:37249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98eda89e24bb4126afa69b37fb3b21d6 is 82, key is hbase:meta,,1/info:regioninfo/1733793266919/Put/seqid=0 2024-12-10T01:17:31,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742382_1558 (size=5672) 2024-12-10T01:17:31,658 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98eda89e24bb4126afa69b37fb3b21d6 2024-12-10T01:17:31,736 DEBUG [M:0;fd1a2b66caf8:37249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/949f9ba4cdd54967b5f9d9a6888cc355 is 2479, key is \x00\x00\x00\x00\x00\x00\x00N/proc:d/1733793357821/Put/seqid=0 2024-12-10T01:17:31,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742383_1559 (size=41523) 2024-12-10T01:17:31,742 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=735.47 KB at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/949f9ba4cdd54967b5f9d9a6888cc355 2024-12-10T01:17:31,745 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 949f9ba4cdd54967b5f9d9a6888cc355 2024-12-10T01:17:31,759 DEBUG [M:0;fd1a2b66caf8:37249 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b225f4846ea24111bc1f756b5a4b422b is 69, key is fd1a2b66caf8,38925,1733793262612/rs:state/1733793265491/Put/seqid=0 2024-12-10T01:17:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073742384_1560 (size=5156) 2024-12-10T01:17:32,163 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2023 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b225f4846ea24111bc1f756b5a4b422b 2024-12-10T01:17:32,173 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98eda89e24bb4126afa69b37fb3b21d6 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/98eda89e24bb4126afa69b37fb3b21d6 2024-12-10T01:17:32,177 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/98eda89e24bb4126afa69b37fb3b21d6, entries=8, sequenceid=2023, filesize=5.5 K 2024-12-10T01:17:32,178 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/949f9ba4cdd54967b5f9d9a6888cc355 as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/949f9ba4cdd54967b5f9d9a6888cc355 2024-12-10T01:17:32,181 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 949f9ba4cdd54967b5f9d9a6888cc355 2024-12-10T01:17:32,181 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/949f9ba4cdd54967b5f9d9a6888cc355, entries=148, sequenceid=2023, filesize=40.5 K 2024-12-10T01:17:32,182 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b225f4846ea24111bc1f756b5a4b422b as hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b225f4846ea24111bc1f756b5a4b422b 2024-12-10T01:17:32,184 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/f155a547-4fec-1e95-96e9-682a773e5126/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b225f4846ea24111bc1f756b5a4b422b, entries=1, sequenceid=2023, filesize=5.0 K 2024-12-10T01:17:32,185 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(3140): Finished flush of dataSize ~736.03 KB/753694, heapSize ~899.16 KB/920736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 952ms, sequenceid=2023, compaction requested=false 2024-12-10T01:17:32,186 INFO [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T01:17:32,186 DEBUG [M:0;fd1a2b66caf8:37249 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733793451232Disabling compacts and flushes for region at 1733793451232Disabling writes for close at 1733793451233 (+1 ms)Obtaining lock to block concurrent updates at 1733793451233Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733793451233Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=753694, getHeapSize=920976, getOffHeapSize=0, getCellsCount=2196 at 1733793451234 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733793451235 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733793451235Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733793451254 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733793451254Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733793451716 (+462 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733793451736 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733793451736Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733793451745 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733793451758 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733793451758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21fa664b: reopening flushed file at 1733793452172 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ef5eb1d: reopening flushed file at 1733793452177 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@392d34b9: reopening flushed file at 1733793452181 (+4 ms)Finished flush of dataSize ~736.03 KB/753694, heapSize ~899.16 KB/920736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 952ms, sequenceid=2023, compaction requested=false at 1733793452185 (+4 ms)Writing region close event to WAL at 1733793452186 (+1 ms)Closed at 1733793452186 2024-12-10T01:17:32,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32797 is added to blk_1073741830_1006 (size=886300) 2024-12-10T01:17:32,195 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T01:17:32,195 INFO [M:0;fd1a2b66caf8:37249 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T01:17:32,195 INFO [M:0;fd1a2b66caf8:37249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37249 2024-12-10T01:17:32,195 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T01:17:32,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T01:17:32,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T01:17:32,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T01:17:32,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T01:17:32,325 INFO [M:0;fd1a2b66caf8:37249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T01:17:32,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T01:17:32,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37249-0x1000d49acf30000, quorum=127.0.0.1:49530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T01:17:32,388 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@53343f2d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T01:17:32,390 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ff8ab72{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T01:17:32,390 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T01:17:32,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a247cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T01:17:32,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@849d63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.log.dir/,STOPPED} 2024-12-10T01:17:32,392 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T01:17:32,392 WARN [BP-1924572727-172.17.0.3-1733793257899 heartbeating to localhost/127.0.0.1:35831 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T01:17:32,392 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T01:17:32,392 WARN [BP-1924572727-172.17.0.3-1733793257899 heartbeating to localhost/127.0.0.1:35831 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1924572727-172.17.0.3-1733793257899 (Datanode Uuid e46bf357-c9e1-452a-b245-f91dfd1874c4) service to localhost/127.0.0.1:35831 2024-12-10T01:17:32,394 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/data/data1/current/BP-1924572727-172.17.0.3-1733793257899 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T01:17:32,394 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/cluster_5a3cbad1-14c5-6f10-1a39-34fe56631f54/data/data2/current/BP-1924572727-172.17.0.3-1733793257899 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T01:17:32,395 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T01:17:32,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f062aca{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T01:17:32,404 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4fee885d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T01:17:32,404 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T01:17:32,404 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@707ebc91{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T01:17:32,404 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b490999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5e9a7853-7ff1-1d7e-c0dc-2e3947251c4d/hadoop.log.dir/,STOPPED} 2024-12-10T01:17:32,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T01:17:32,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down